Added hindsight_liteLLM implementation (#17)
* Added hindsight_liteLLM implementation * Add instructions for entity vs bank id * Add another line about entity * Address PR review comments and enhance litellm integration - Remove deprecated limit parameter from recall() and arecall() functions since Hindsight uses budget/max_tokens for result control - Remove dead MODEL_MAX_OUTPUT_TOKENS dict and max_output_tokens property from LLMProvider (superseded by hardcoded max_completion_tokens) - Add test-litellm-integration job to CI workflow - Add reflect API support with use_reflect config option - Add verbose mode debug info via get_last_injection_debug() - Add entity_id support for multi-user memory isolation - Add retain() and reflect() wrapper functions - Update docstrings and examples 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Make max_memories optional to allow unlimited memory injection - Change max_memories default from 10 to None (no limit) - When max_memories is None, all results from the API are used - Fix recall result handling to properly detect list vs object return - Update wrappers (OpenAI, Anthropic) with same optional behavior This allows users to control memory limits via max_memory_tokens and recall_budget without an artificial count limit. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Remove entity_id from hindsight_litellm; add gpt-4o token cap Multi-user support now uses separate bank_ids per user instead of entity_id scoping (e.g., bank_id=f"user-{user_id}"). This simplifies the API and aligns with the Hindsight architecture. Also fixes max_completion_tokens error for gpt-4o models by capping the value at 16384 (gpt-4o's limit) instead of sending the default 65000 which exceeds the model's supported maximum. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix dark mode styling across Control Plane UI components Improvements to ensure proper text visibility and contrast in both light and dark modes: - Add global CSS rules for datetime-local calendar picker icon visibility using filter: invert() for both light (0.5) and dark (1) modes - Fix text colors in dialog components to use theme-aware foreground colors - Update memory detail panel, document/chunk modals, and data views to use proper dark mode text classes (text-foreground, text-card-foreground) - Fix form labels, headings, and content text in bank selector dialogs - Update entities view and documents view table styling for dark mode - Bump package versions to 0.1.4 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Remove session_id feature and add How It Works section to README - Remove session_id and session management (new_session, set_session, get_session) from config.py, callbacks.py, and __init__.py - Session management was a client-only abstraction not backed by core API - Add "How It Works" section to README with visual flow diagram - Update README to remove session management documentation 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix readme example * Add dark mode again --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
DK09876
GitHub
parent
dfea4dbe15
commit
dfccbf29f1
20 changed files with 3772 additions and 70 deletions
29
.github/workflows/test.yml
vendored
29
.github/workflows/test.yml
vendored
|
|
@ -441,3 +441,32 @@ jobs:
|
|||
run: |
|
||||
echo "=== API Server Logs ==="
|
||||
cat /tmp/api-server.log || echo "No API server log found"
|
||||
|
||||
test-litellm-integration:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
prune-cache: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: ".python-version"
|
||||
|
||||
- name: Build litellm integration
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv build
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv sync --extra dev
|
||||
|
||||
- name: Run tests
|
||||
working-directory: ./hindsight-integrations/litellm
|
||||
run: uv run pytest tests -v
|
||||
|
|
@ -175,9 +175,13 @@ class LLMProvider:
|
|||
is_reasoning_model = any(x in model_lower for x in ["gpt-5", "o1", "o3"])
|
||||
|
||||
# For GPT-4 and GPT-4.1 models, cap max_completion_tokens to 32000
|
||||
# For GPT-4o models, cap to 16384
|
||||
is_gpt4_model = any(x in model_lower for x in ["gpt-4.1", "gpt-4-"])
|
||||
is_gpt4o_model = "gpt-4o" in model_lower
|
||||
if max_completion_tokens is not None:
|
||||
if is_gpt4_model and max_completion_tokens > 32000:
|
||||
if is_gpt4o_model and max_completion_tokens > 16384:
|
||||
max_completion_tokens = 16384
|
||||
elif is_gpt4_model and max_completion_tokens > 32000:
|
||||
max_completion_tokens = 32000
|
||||
# For reasoning models, max_completion_tokens includes reasoning + output tokens
|
||||
# Enforce minimum of 16000 to ensure enough space for both
|
||||
|
|
|
|||
|
|
@ -54,7 +54,9 @@ class EmbeddedPostgres:
|
|||
loop = asyncio.get_event_loop()
|
||||
info = await loop.run_in_executor(None, pg0.start)
|
||||
logger.info(f"PostgreSQL started on port {self.port}")
|
||||
return info.uri
|
||||
# Construct URI manually since pg0-embedded may return None
|
||||
uri = info.uri if info and info.uri else f"postgresql://{self.username}:{self.password}@localhost:{self.port}/{self.database}"
|
||||
return uri
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
if attempt < max_retries:
|
||||
|
|
@ -89,9 +91,9 @@ class EmbeddedPostgres:
|
|||
pg0 = self._get_pg0()
|
||||
loop = asyncio.get_event_loop()
|
||||
info = await loop.run_in_executor(None, pg0.info)
|
||||
if info is None or not info.running:
|
||||
raise RuntimeError("PostgreSQL server is not running or URI not available")
|
||||
return info.uri
|
||||
# Construct URI manually since pg0-embedded may return None
|
||||
uri = info.uri if info and info.uri else f"postgresql://{self.username}:{self.password}@localhost:{self.port}/{self.database}"
|
||||
return uri
|
||||
|
||||
async def is_running(self) -> bool:
|
||||
"""Check if the PostgreSQL server is currently running."""
|
||||
|
|
|
|||
|
|
@ -180,4 +180,13 @@ code, pre {
|
|||
-webkit-background-clip: text;
|
||||
-webkit-text-fill-color: transparent;
|
||||
background-clip: text;
|
||||
}
|
||||
|
||||
/* Fix datetime-local calendar icon visibility in both light and dark modes */
|
||||
input[type="datetime-local"]::-webkit-calendar-picker-indicator {
|
||||
filter: invert(0.5);
|
||||
}
|
||||
|
||||
.dark input[type="datetime-local"]::-webkit-calendar-picker-indicator {
|
||||
filter: invert(1);
|
||||
}
|
||||
|
|
@ -239,7 +239,7 @@ export function BankProfileView() {
|
|||
<div className="flex gap-2">
|
||||
{editMode ? (
|
||||
<>
|
||||
<Button onClick={handleCancel} variant="outline" disabled={saving}>
|
||||
<Button onClick={handleCancel} variant="secondary" disabled={saving}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button onClick={handleSave} disabled={saving}>
|
||||
|
|
@ -258,7 +258,7 @@ export function BankProfileView() {
|
|||
</>
|
||||
) : (
|
||||
<>
|
||||
<Button onClick={loadData} variant="outline" size="sm">
|
||||
<Button onClick={loadData} variant="secondary" size="sm">
|
||||
<RefreshCw className="w-4 h-4 mr-2" />
|
||||
Refresh
|
||||
</Button>
|
||||
|
|
|
|||
|
|
@ -266,7 +266,7 @@ function BankSelectorInner() {
|
|||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setCreateDialogOpen(false);
|
||||
setNewBankId('');
|
||||
|
|
@ -295,7 +295,7 @@ function BankSelectorInner() {
|
|||
</DialogHeader>
|
||||
<div className="py-4 space-y-4">
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Content *</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Content *</label>
|
||||
<Textarea
|
||||
value={docContent}
|
||||
onChange={(e) => setDocContent(e.target.value)}
|
||||
|
|
@ -306,7 +306,7 @@ function BankSelectorInner() {
|
|||
</div>
|
||||
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Context</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Context</label>
|
||||
<Input
|
||||
type="text"
|
||||
value={docContext}
|
||||
|
|
@ -317,16 +317,17 @@ function BankSelectorInner() {
|
|||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Event Date</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Event Date</label>
|
||||
<Input
|
||||
type="datetime-local"
|
||||
value={docEventDate}
|
||||
onChange={(e) => setDocEventDate(e.target.value)}
|
||||
className="text-foreground"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="font-bold block mb-1 text-sm">Document ID</label>
|
||||
<label className="font-bold block mb-1 text-sm text-foreground">Document ID</label>
|
||||
<Input
|
||||
type="text"
|
||||
value={docDocumentId}
|
||||
|
|
@ -342,7 +343,7 @@ function BankSelectorInner() {
|
|||
checked={docAsync}
|
||||
onCheckedChange={(checked) => setDocAsync(checked as boolean)}
|
||||
/>
|
||||
<label htmlFor="async-doc" className="text-sm cursor-pointer">
|
||||
<label htmlFor="async-doc" className="text-sm cursor-pointer text-foreground">
|
||||
Process in background (async)
|
||||
</label>
|
||||
</div>
|
||||
|
|
@ -353,7 +354,7 @@ function BankSelectorInner() {
|
|||
</div>
|
||||
<DialogFooter>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
onClick={() => {
|
||||
setDocDialogOpen(false);
|
||||
setDocContent('');
|
||||
|
|
|
|||
|
|
@ -480,7 +480,7 @@ export function DataView({ factType }: DataViewProps) {
|
|||
}`}
|
||||
>
|
||||
<TableCell className="py-2">
|
||||
<div className="line-clamp-2 text-sm leading-snug">{row.text}</div>
|
||||
<div className="line-clamp-2 text-sm leading-snug text-foreground">{row.text}</div>
|
||||
{row.context && (
|
||||
<div className="text-xs text-muted-foreground mt-0.5 truncate">{row.context}</div>
|
||||
)}
|
||||
|
|
@ -506,10 +506,10 @@ export function DataView({ factType }: DataViewProps) {
|
|||
<span className="text-xs text-muted-foreground">-</span>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs py-2">
|
||||
<TableCell className="text-xs py-2 text-foreground">
|
||||
{occurredDisplay || <span className="text-muted-foreground">-</span>}
|
||||
</TableCell>
|
||||
<TableCell className="text-xs py-2">
|
||||
<TableCell className="text-xs py-2 text-foreground">
|
||||
{mentionedDisplay || <span className="text-muted-foreground">-</span>}
|
||||
</TableCell>
|
||||
<TableCell className="py-2">
|
||||
|
|
@ -519,7 +519,7 @@ export function DataView({ factType }: DataViewProps) {
|
|||
copyToClipboard(row.id);
|
||||
}}
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
className="h-6 w-6 p-0"
|
||||
title="Copy ID"
|
||||
>
|
||||
|
|
@ -799,7 +799,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
{/* Zoom controls */}
|
||||
<div className="flex items-center border border-border rounded mr-2">
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={zoomOut}
|
||||
disabled={granularity === 'year'}
|
||||
|
|
@ -808,11 +808,11 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
>
|
||||
<ZoomOut className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-[10px] px-2 min-w-[50px] text-center border-x border-border">
|
||||
<span className="text-[10px] px-2 min-w-[50px] text-center border-x border-border text-foreground">
|
||||
{granularityLabels[granularity]}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={zoomIn}
|
||||
disabled={granularity === 'day'}
|
||||
|
|
@ -826,7 +826,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
{/* Navigation controls */}
|
||||
<div className="flex items-center border border-border rounded">
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(0)}
|
||||
disabled={timelineGroups.length <= 1}
|
||||
|
|
@ -836,7 +836,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
<ChevronsLeft className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(currentIndex - 1)}
|
||||
disabled={currentIndex === 0}
|
||||
|
|
@ -845,11 +845,11 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
>
|
||||
<ChevronLeft className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-[10px] px-2 min-w-[60px] text-center border-x border-border">
|
||||
<span className="text-[10px] px-2 min-w-[60px] text-center border-x border-border text-foreground">
|
||||
{currentIndex + 1} / {timelineGroups.length}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(currentIndex + 1)}
|
||||
disabled={currentIndex >= timelineGroups.length - 1}
|
||||
|
|
@ -859,7 +859,7 @@ function TimelineView({ data, filteredRows }: { data: any; filteredRows: any[] }
|
|||
<ChevronRight className="h-3 w-3" />
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => scrollToGroup(timelineGroups.length - 1)}
|
||||
disabled={timelineGroups.length <= 1}
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Document ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">{data.id}</div>
|
||||
<div className="text-sm font-mono break-all text-foreground">{data.id}</div>
|
||||
</div>
|
||||
{data.created_at && (
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
|
|
@ -102,7 +102,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Created
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{new Date(data.created_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -110,7 +110,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Memory Units
|
||||
</div>
|
||||
<div className="text-sm">{data.memory_unit_count}</div>
|
||||
<div className="text-sm text-foreground">{data.memory_unit_count}</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -119,7 +119,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Text Length
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{data.original_text.length.toLocaleString()} characters
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -132,7 +132,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
Original Text
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg border border-border max-h-[300px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono text-foreground">
|
||||
{data.original_text}
|
||||
</pre>
|
||||
</div>
|
||||
|
|
@ -146,7 +146,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Chunk ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">
|
||||
<div className="text-sm font-mono break-all text-foreground">
|
||||
{data.chunk_id}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -155,7 +155,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Document ID
|
||||
</div>
|
||||
<div className="text-sm font-mono break-all">
|
||||
<div className="text-sm font-mono break-all text-foreground">
|
||||
{data.document_id}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -163,7 +163,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Chunk Index
|
||||
</div>
|
||||
<div className="text-sm">{data.chunk_index}</div>
|
||||
<div className="text-sm text-foreground">{data.chunk_index}</div>
|
||||
</div>
|
||||
</div>
|
||||
{data.created_at && (
|
||||
|
|
@ -171,7 +171,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Created
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{new Date(data.created_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -181,7 +181,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
<div className="text-xs font-bold text-muted-foreground uppercase mb-1">
|
||||
Text Length
|
||||
</div>
|
||||
<div className="text-sm">
|
||||
<div className="text-sm text-foreground">
|
||||
{data.chunk_text.length.toLocaleString()} characters
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -194,7 +194,7 @@ export function DocumentChunkModal({ type, id, onClose }: DocumentChunkModalProp
|
|||
Chunk Text
|
||||
</div>
|
||||
<div className="p-4 bg-muted rounded-lg border border-border max-h-[300px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono text-foreground">
|
||||
{data.chunk_text}
|
||||
</pre>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -122,17 +122,17 @@ export function DocumentsView() {
|
|||
className={`cursor-pointer hover:bg-muted/50 ${selectedDocument?.id === doc.id ? 'bg-primary/10' : ''}`}
|
||||
onClick={() => viewDocumentText(doc.id)}
|
||||
>
|
||||
<TableCell title={doc.id}>
|
||||
<TableCell title={doc.id} className="text-card-foreground">
|
||||
{doc.id.length > 30 ? doc.id.substring(0, 30) + '...' : doc.id}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TableCell className="text-card-foreground">
|
||||
{doc.created_at ? new Date(doc.created_at).toLocaleString() : 'N/A'}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TableCell className="text-card-foreground">
|
||||
{doc.retain_params?.context || '-'}
|
||||
</TableCell>
|
||||
<TableCell>{doc.text_length?.toLocaleString()} chars</TableCell>
|
||||
<TableCell>{doc.memory_unit_count}</TableCell>
|
||||
<TableCell className="text-card-foreground">{doc.text_length?.toLocaleString()} chars</TableCell>
|
||||
<TableCell className="text-card-foreground">{doc.memory_unit_count}</TableCell>
|
||||
<TableCell>
|
||||
<Button
|
||||
onClick={(e) => {
|
||||
|
|
@ -140,7 +140,7 @@ export function DocumentsView() {
|
|||
viewDocumentText(doc.id);
|
||||
}}
|
||||
size="sm"
|
||||
variant={selectedDocument?.id === doc.id ? 'default' : 'outline'}
|
||||
variant={selectedDocument?.id === doc.id ? 'default' : 'secondary'}
|
||||
title="View original text"
|
||||
>
|
||||
View Text
|
||||
|
|
@ -171,7 +171,7 @@ export function DocumentsView() {
|
|||
<p className="text-sm text-muted-foreground mt-1">Original document text and metadata</p>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={() => setSelectedDocument(null)}
|
||||
className="h-9 px-3 gap-2"
|
||||
|
|
@ -193,7 +193,7 @@ export function DocumentsView() {
|
|||
{/* Document ID */}
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Document ID</div>
|
||||
<code className="text-sm font-mono break-all text-foreground">{selectedDocument.id}</code>
|
||||
<div className="text-sm font-mono break-all text-card-foreground">{selectedDocument.id}</div>
|
||||
</div>
|
||||
|
||||
{/* Created & Memory Units */}
|
||||
|
|
@ -201,11 +201,11 @@ export function DocumentsView() {
|
|||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Created</div>
|
||||
<div className="text-sm font-medium text-foreground">{new Date(selectedDocument.created_at).toLocaleString()}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{new Date(selectedDocument.created_at).toLocaleString()}</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Memory Units</div>
|
||||
<div className="text-sm font-medium text-foreground">{selectedDocument.memory_unit_count}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{selectedDocument.memory_unit_count}</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -214,7 +214,7 @@ export function DocumentsView() {
|
|||
{selectedDocument.original_text && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Text Length</div>
|
||||
<div className="text-sm font-medium text-foreground">{selectedDocument.original_text.length.toLocaleString()} characters</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{selectedDocument.original_text.length.toLocaleString()} characters</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
|
@ -222,7 +222,7 @@ export function DocumentsView() {
|
|||
{selectedDocument.retain_params && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Retain Parameters</div>
|
||||
<div className="text-sm space-y-2">
|
||||
<div className="text-sm space-y-2 text-card-foreground">
|
||||
{selectedDocument.retain_params.context && (
|
||||
<div><span className="font-semibold">Context:</span> {selectedDocument.retain_params.context}</div>
|
||||
)}
|
||||
|
|
@ -232,7 +232,7 @@ export function DocumentsView() {
|
|||
{selectedDocument.retain_params.metadata && (
|
||||
<div className="mt-2">
|
||||
<span className="font-semibold">Metadata:</span>
|
||||
<pre className="mt-1 text-xs bg-background p-2 rounded">{JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}</pre>
|
||||
<pre className="mt-1 text-xs bg-background p-2 rounded text-card-foreground">{JSON.stringify(selectedDocument.retain_params.metadata, null, 2)}</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -244,7 +244,7 @@ export function DocumentsView() {
|
|||
<div>
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Original Text</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg border border-border max-h-[400px] overflow-y-auto">
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono leading-relaxed text-foreground">{selectedDocument.original_text}</pre>
|
||||
<pre className="text-sm whitespace-pre-wrap font-mono leading-relaxed text-card-foreground">{selectedDocument.original_text}</pre>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -126,10 +126,10 @@ export function EntitiesView() {
|
|||
selectedEntity?.id === entity.id ? 'bg-primary/10' : ''
|
||||
}`}
|
||||
>
|
||||
<TableCell className="font-medium">{entity.canonical_name}</TableCell>
|
||||
<TableCell>{entity.mention_count}</TableCell>
|
||||
<TableCell>{formatDate(entity.first_seen)}</TableCell>
|
||||
<TableCell>{formatDate(entity.last_seen)}</TableCell>
|
||||
<TableCell className="font-medium text-card-foreground">{entity.canonical_name}</TableCell>
|
||||
<TableCell className="text-card-foreground">{entity.mention_count}</TableCell>
|
||||
<TableCell className="text-card-foreground">{formatDate(entity.first_seen)}</TableCell>
|
||||
<TableCell className="text-card-foreground">{formatDate(entity.last_seen)}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
|
|
@ -154,7 +154,7 @@ export function EntitiesView() {
|
|||
{/* Header */}
|
||||
<div className="flex justify-between items-center mb-6 pb-4 border-b border-border">
|
||||
<div>
|
||||
<h3 className="text-xl font-bold text-foreground">{selectedEntity.canonical_name}</h3>
|
||||
<h3 className="text-xl font-bold text-card-foreground">{selectedEntity.canonical_name}</h3>
|
||||
<p className="text-sm text-muted-foreground mt-1">Entity details</p>
|
||||
</div>
|
||||
<Button
|
||||
|
|
@ -172,11 +172,11 @@ export function EntitiesView() {
|
|||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Mentions</div>
|
||||
<div className="text-lg font-semibold text-foreground">{selectedEntity.mention_count}</div>
|
||||
<div className="text-lg font-semibold text-card-foreground">{selectedEntity.mention_count}</div>
|
||||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">First Seen</div>
|
||||
<div className="text-sm font-medium text-foreground">{formatDate(selectedEntity.first_seen)}</div>
|
||||
<div className="text-sm font-medium text-card-foreground">{formatDate(selectedEntity.first_seen)}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -206,7 +206,7 @@ export function EntitiesView() {
|
|||
<ul className="space-y-2">
|
||||
{selectedEntity.observations.map((obs, idx) => (
|
||||
<li key={idx} className="p-3 bg-muted/50 rounded-lg">
|
||||
<div className="text-sm text-foreground">{obs.text}</div>
|
||||
<div className="text-sm text-card-foreground">{obs.text}</div>
|
||||
{obs.mentioned_at && (
|
||||
<div className="text-xs text-muted-foreground mt-2">
|
||||
{formatDate(obs.mentioned_at)}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ export function MemoryDetailPanel({
|
|||
<p className="text-sm text-muted-foreground mt-1">Full memory content and metadata</p>
|
||||
</div>
|
||||
<Button
|
||||
variant="ghost"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={onClose}
|
||||
className="h-8 w-8 p-0"
|
||||
|
|
@ -80,14 +80,14 @@ export function MemoryDetailPanel({
|
|||
{/* Full Text */}
|
||||
<div>
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Full Text</div>
|
||||
<div className="text-sm whitespace-pre-wrap leading-relaxed">{memory.text}</div>
|
||||
<div className="text-sm whitespace-pre-wrap leading-relaxed text-foreground">{memory.text}</div>
|
||||
</div>
|
||||
|
||||
{/* Context */}
|
||||
{memory.context && (
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Context</div>
|
||||
<div className="text-sm">{memory.context}</div>
|
||||
<div className="text-sm text-foreground">{memory.context}</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
|
@ -95,7 +95,7 @@ export function MemoryDetailPanel({
|
|||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Occurred</div>
|
||||
<div className="text-sm font-medium">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{memory.occurred_start
|
||||
? new Date(memory.occurred_start).toLocaleString()
|
||||
: 'N/A'}
|
||||
|
|
@ -103,7 +103,7 @@ export function MemoryDetailPanel({
|
|||
</div>
|
||||
<div className="p-4 bg-muted/50 rounded-lg">
|
||||
<div className="text-xs font-bold text-muted-foreground uppercase mb-2">Mentioned</div>
|
||||
<div className="text-sm font-medium">
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{memory.mentioned_at
|
||||
? new Date(memory.mentioned_at).toLocaleString()
|
||||
: 'N/A'}
|
||||
|
|
@ -159,7 +159,7 @@ export function MemoryDetailPanel({
|
|||
{memory.document_id && (
|
||||
<Button
|
||||
onClick={() => openDocumentModal(memory.document_id)}
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className="flex-1"
|
||||
>
|
||||
View Document
|
||||
|
|
@ -168,7 +168,7 @@ export function MemoryDetailPanel({
|
|||
{memory.chunk_id && (
|
||||
<Button
|
||||
onClick={() => openChunkModal(memory.chunk_id)}
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className="flex-1"
|
||||
>
|
||||
View Chunk
|
||||
|
|
@ -300,7 +300,7 @@ export function MemoryDetailPanel({
|
|||
<Button
|
||||
onClick={() => openDocumentModal(memory.document_id)}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className={`flex-1 ${compact ? 'h-7 text-xs' : ''}`}
|
||||
>
|
||||
View Document
|
||||
|
|
@ -310,7 +310,7 @@ export function MemoryDetailPanel({
|
|||
<Button
|
||||
onClick={() => openChunkModal(memory.chunk_id)}
|
||||
size="sm"
|
||||
variant="outline"
|
||||
variant="secondary"
|
||||
className={`flex-1 ${compact ? 'h-7 text-xs' : ''}`}
|
||||
>
|
||||
View Chunk
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ const DialogContent = React.forwardRef<
|
|||
{...props}
|
||||
>
|
||||
{children}
|
||||
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground">
|
||||
<DialogPrimitive.Close className="absolute right-4 top-4 rounded-sm ring-offset-background transition-opacity hover:opacity-80 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground text-foreground">
|
||||
<X className="h-4 w-4" />
|
||||
<span className="sr-only">Close</span>
|
||||
</DialogPrimitive.Close>
|
||||
|
|
@ -88,7 +88,7 @@ const DialogTitle = React.forwardRef<
|
|||
<DialogPrimitive.Title
|
||||
ref={ref}
|
||||
className={cn(
|
||||
"text-lg font-semibold leading-none tracking-tight",
|
||||
"text-lg font-semibold leading-none tracking-tight text-foreground",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
|
|
|
|||
437
hindsight-integrations/litellm/README.md
Normal file
437
hindsight-integrations/litellm/README.md
Normal file
|
|
@ -0,0 +1,437 @@
|
|||
# hindsight-litellm
|
||||
|
||||
Universal LLM memory integration via LiteLLM. Add persistent memory to any LLM application with just a few lines of code.
|
||||
|
||||
## Features
|
||||
|
||||
- **Universal LLM Support** - Works with 100+ LLM providers via LiteLLM (OpenAI, Anthropic, Groq, Azure, AWS Bedrock, Google Vertex AI, and more)
|
||||
- **Simple Integration** - Just configure, enable, and use `hindsight_litellm.completion()`
|
||||
- **Automatic Memory Injection** - Relevant memories are injected into prompts before LLM calls
|
||||
- **Automatic Conversation Storage** - Conversations are stored to Hindsight for future recall
|
||||
- **Two Memory Modes** - Choose between `reflect` (synthesized context) or `recall` (raw memory retrieval)
|
||||
- **Direct Memory APIs** - Query, synthesize, and store memories manually
|
||||
- **Native Client Wrappers** - Alternative wrappers for OpenAI and Anthropic SDKs
|
||||
- **Debug Mode** - Inspect exactly what memories are being injected
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install hindsight-litellm
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
# Configure and enable memory integration
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# Use the convenience wrapper - memory is automatically injected and stored
|
||||
response = hindsight_litellm.completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What did we discuss about AI?"}]
|
||||
)
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
Here's what happens under the hood when you call `completion()`:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 1. YOUR CODE │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ response = hindsight_litellm.completion( │
|
||||
│ model="gpt-4o-mini", │
|
||||
│ messages=[{"role": "user", "content": "Help me with my Python project"}]│
|
||||
│ ) │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 2. MEMORY RETRIEVAL (before LLM call) │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # hindsight_litellm queries Hindsight for relevant memories │
|
||||
│ │
|
||||
│ # If use_reflect=False (default) - raw memories: │
|
||||
│ memories = hindsight.recall(query="Help me with my Python project") │
|
||||
│ # Returns: ["User prefers pytest", "User is building a FastAPI app", ...] │
|
||||
│ │
|
||||
│ # If use_reflect=True - synthesized context: │
|
||||
│ context = hindsight.reflect(query="Help me with my Python project") │
|
||||
│ # Returns: "The user is an experienced Python developer working on..." │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 3. PROMPT INJECTION │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # Memories are injected into the system message: │
|
||||
│ │
|
||||
│ messages = [ │
|
||||
│ {"role": "system", "content": """ │
|
||||
│ # Relevant Memories │
|
||||
│ 1. [WORLD] User prefers pytest for testing │
|
||||
│ 2. [WORLD] User is building a FastAPI app │
|
||||
│ 3. [OPINION] User likes type hints │
|
||||
│ """}, │
|
||||
│ {"role": "user", "content": "Help me with my Python project"} │
|
||||
│ ] │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 4. LLM CALL │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # The enriched prompt is sent to the LLM │
|
||||
│ response = litellm.completion(model="gpt-4o-mini", messages=messages) │
|
||||
│ │
|
||||
│ # LLM now has context and can give personalized responses like: │
|
||||
│ # "Since you're working on your FastAPI app, here's how to add tests │
|
||||
│ # with pytest..." │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 5. CONVERSATION STORAGE (after LLM call) │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # The conversation is stored to Hindsight for future recall │
|
||||
│ hindsight.retain( │
|
||||
│ content="User: Help me with my Python project\n" │
|
||||
│ "Assistant: Since you're working on FastAPI..." │
|
||||
│ ) │
|
||||
│ # Hindsight extracts facts: "User asked about Python project help" │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 6. RESPONSE RETURNED │
|
||||
│ ───────────────────────────────────────────────────────────────────────── │
|
||||
│ # You receive the response as normal │
|
||||
│ print(response.choices[0].message.content) │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
The memory injection and storage happen automatically - you just use `completion()` as normal.
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```python
|
||||
hindsight_litellm.configure(
|
||||
# Required
|
||||
hindsight_api_url="http://localhost:8888", # Hindsight API server URL
|
||||
bank_id="my-agent", # Memory bank ID
|
||||
|
||||
api_key="your-api-key", # Optional API key for authentication
|
||||
|
||||
# Optional - Memory behavior
|
||||
store_conversations=True, # Store conversations after LLM calls
|
||||
inject_memories=True, # Inject relevant memories into prompts
|
||||
use_reflect=False, # Use reflect API (synthesized) vs recall (raw memories)
|
||||
reflect_include_facts=False, # Include source facts with reflect responses
|
||||
max_memories=None, # Maximum memories to inject (None = unlimited)
|
||||
max_memory_tokens=4096, # Maximum tokens for memory context
|
||||
recall_budget="mid", # Recall budget: "low", "mid", "high"
|
||||
fact_types=["world", "agent"], # Filter fact types to inject
|
||||
|
||||
# Optional - Bank Configuration
|
||||
bank_name="My Agent", # Human-readable display name for the memory bank
|
||||
background="This agent...", # Instructions guiding what Hindsight should remember (see below)
|
||||
|
||||
# Optional - Advanced
|
||||
injection_mode="system_message", # or "prepend_user"
|
||||
excluded_models=["gpt-3.5*"], # Exclude certain models
|
||||
verbose=True, # Enable verbose logging and debug info
|
||||
)
|
||||
```
|
||||
|
||||
### Bank Configuration: background and bank_name
|
||||
|
||||
The `background` and `bank_name` parameters configure the memory bank itself. When provided, `configure()` will automatically create or update the bank with these settings.
|
||||
|
||||
- **bank_name**: A human-readable display name for the memory bank. Useful for identifying banks in the Hindsight UI or when managing multiple banks.
|
||||
|
||||
- **background**: Instructions that guide Hindsight on what information is important to extract and remember from conversations. This influences memory extraction during the `retain` operation and can affect how the bank's "disposition" (skepticism, literalism, empathy) is calibrated.
|
||||
|
||||
```python
|
||||
# Example: Customer support routing agent
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="support-router",
|
||||
bank_name="Customer Support Router",
|
||||
background="""This agent routes customer support requests to the appropriate team.
|
||||
Remember which types of issues should go to which teams (billing, technical, sales).
|
||||
Track customer preferences for communication channels and past issue resolutions.
|
||||
Note any escalation patterns or VIP customers who need special handling.""",
|
||||
)
|
||||
```
|
||||
|
||||
### Memory Modes: Reflect vs Recall
|
||||
|
||||
- **Recall mode** (`use_reflect=False`, default): Retrieves raw memory facts and injects them as a numbered list. Best when you need precise, individual memories.
|
||||
- **Reflect mode** (`use_reflect=True`): Synthesizes memories into a coherent context paragraph. Best for natural, conversational memory context.
|
||||
|
||||
```python
|
||||
# Recall mode - raw memories
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=False, # Default
|
||||
)
|
||||
# Injects: "1. [WORLD] User prefers Python\n2. [OPINION] User dislikes Java..."
|
||||
|
||||
# Reflect mode - synthesized context
|
||||
hindsight_litellm.configure(
|
||||
bank_id="my-agent",
|
||||
use_reflect=True,
|
||||
)
|
||||
# Injects: "Based on previous conversations, the user is a Python developer who..."
|
||||
```
|
||||
|
||||
## Multi-Provider Support
|
||||
|
||||
Works with any LiteLLM-supported provider:
|
||||
|
||||
```python
|
||||
import hindsight_litellm
|
||||
|
||||
hindsight_litellm.configure(
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
bank_id="my-agent",
|
||||
)
|
||||
hindsight_litellm.enable()
|
||||
|
||||
# OpenAI
|
||||
hindsight_litellm.completion(model="gpt-4o", messages=[...])
|
||||
|
||||
# Anthropic
|
||||
hindsight_litellm.completion(model="claude-3-5-sonnet-20241022", messages=[...])
|
||||
|
||||
# Groq
|
||||
hindsight_litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...])
|
||||
|
||||
# Azure OpenAI
|
||||
hindsight_litellm.completion(model="azure/gpt-4", messages=[...])
|
||||
|
||||
# AWS Bedrock
|
||||
hindsight_litellm.completion(model="bedrock/anthropic.claude-3", messages=[...])
|
||||
|
||||
# Google Vertex AI
|
||||
hindsight_litellm.completion(model="vertex_ai/gemini-pro", messages=[...])
|
||||
```
|
||||
|
||||
## Direct Memory APIs
|
||||
|
||||
### Recall - Query raw memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, recall
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Query memories
|
||||
memories = recall("what projects am I working on?", budget="mid")
|
||||
for m in memories:
|
||||
print(f"- [{m.fact_type}] {m.text}")
|
||||
|
||||
# Output:
|
||||
# - [world] User is building a FastAPI project
|
||||
# - [opinion] User prefers Python over JavaScript
|
||||
```
|
||||
|
||||
### Reflect - Get synthesized context
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, reflect
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Get synthesized memory context
|
||||
result = reflect("what do you know about the user's preferences?")
|
||||
print(result.text)
|
||||
|
||||
# Output:
|
||||
# "Based on our conversations, the user prefers Python for backend development..."
|
||||
```
|
||||
|
||||
### Retain - Store memories
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, retain
|
||||
|
||||
configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
|
||||
# Store a memory
|
||||
result = retain(
|
||||
content="User mentioned they're working on a machine learning project",
|
||||
context="Discussion about current projects",
|
||||
)
|
||||
print(f"Retained successfully: {result.success}, items: {result.items_count}")
|
||||
```
|
||||
|
||||
### Async APIs
|
||||
|
||||
```python
|
||||
from hindsight_litellm import arecall, areflect, aretain
|
||||
|
||||
# Async versions of all memory APIs
|
||||
memories = await arecall("what do you know about me?")
|
||||
context = await areflect("summarize user preferences")
|
||||
result = await aretain(content="New information to remember")
|
||||
```
|
||||
|
||||
## Native Client Wrappers
|
||||
|
||||
Alternative to LiteLLM callbacks for direct SDK integration:
|
||||
|
||||
### OpenAI Wrapper
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
from hindsight_litellm import wrap_openai
|
||||
|
||||
client = OpenAI()
|
||||
wrapped = wrap_openai(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "What do you know about me?"}]
|
||||
)
|
||||
```
|
||||
|
||||
### Anthropic Wrapper
|
||||
|
||||
```python
|
||||
from anthropic import Anthropic
|
||||
from hindsight_litellm import wrap_anthropic
|
||||
|
||||
client = Anthropic()
|
||||
wrapped = wrap_anthropic(
|
||||
client,
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
response = wrapped.messages.create(
|
||||
model="claude-3-5-sonnet-20241022",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "Hello!"}]
|
||||
)
|
||||
```
|
||||
|
||||
## Debug Mode
|
||||
|
||||
When `verbose=True`, you can inspect exactly what memories are being injected:
|
||||
|
||||
```python
|
||||
from hindsight_litellm import configure, enable, completion, get_last_injection_debug
|
||||
|
||||
configure(
|
||||
bank_id="my-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
verbose=True,
|
||||
use_reflect=True,
|
||||
)
|
||||
enable()
|
||||
|
||||
response = completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "What's my favorite color?"}]
|
||||
)
|
||||
|
||||
# Inspect what was injected
|
||||
debug = get_last_injection_debug()
|
||||
if debug:
|
||||
print(f"Mode: {debug.mode}") # "reflect" or "recall"
|
||||
print(f"Injected: {debug.injected}") # True/False
|
||||
print(f"Results: {debug.results_count}")
|
||||
print(f"Memory context:\n{debug.memory_context}")
|
||||
if debug.error:
|
||||
print(f"Error: {debug.error}")
|
||||
```
|
||||
|
||||
## Context Manager
|
||||
|
||||
```python
|
||||
from hindsight_litellm import hindsight_memory
|
||||
import litellm
|
||||
|
||||
with hindsight_memory(bank_id="user-123"):
|
||||
response = litellm.completion(model="gpt-4", messages=[...])
|
||||
# Memory integration automatically disabled after context
|
||||
```
|
||||
|
||||
## Disabling and Cleanup
|
||||
|
||||
```python
|
||||
from hindsight_litellm import disable, cleanup
|
||||
|
||||
# Temporarily disable memory integration
|
||||
disable()
|
||||
|
||||
# Clean up all resources (call when shutting down)
|
||||
cleanup()
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Main Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `configure(...)` | Configure global Hindsight settings |
|
||||
| `enable()` | Enable memory integration with LiteLLM |
|
||||
| `disable()` | Disable memory integration |
|
||||
| `is_enabled()` | Check if memory integration is enabled |
|
||||
| `cleanup()` | Clean up all resources |
|
||||
|
||||
### Configuration Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_config()` | Get current configuration |
|
||||
| `is_configured()` | Check if Hindsight is configured |
|
||||
| `reset_config()` | Reset configuration to defaults |
|
||||
|
||||
### Memory Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `recall(query, ...)` | Synchronously query raw memories |
|
||||
| `arecall(query, ...)` | Asynchronously query raw memories |
|
||||
| `reflect(query, ...)` | Synchronously get synthesized memory context |
|
||||
| `areflect(query, ...)` | Asynchronously get synthesized memory context |
|
||||
| `retain(content, ...)` | Synchronously store a memory |
|
||||
| `aretain(content, ...)` | Asynchronously store a memory |
|
||||
|
||||
### Debug Functions
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `get_last_injection_debug()` | Get debug info from last memory injection |
|
||||
| `clear_injection_debug()` | Clear stored debug info |
|
||||
|
||||
### Client Wrappers
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `wrap_openai(client, ...)` | Wrap OpenAI client with memory |
|
||||
| `wrap_anthropic(client, ...)` | Wrap Anthropic client with memory |
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python >= 3.10
|
||||
- litellm >= 1.40.0
|
||||
- A running Hindsight API server
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
817
hindsight-integrations/litellm/hindsight_litellm/__init__.py
Normal file
817
hindsight-integrations/litellm/hindsight_litellm/__init__.py
Normal file
|
|
@ -0,0 +1,817 @@
|
|||
"""Hindsight-LiteLLM: Universal LLM memory integration via LiteLLM.
|
||||
|
||||
This package provides automatic memory integration for any LLM provider
|
||||
supported by LiteLLM (100+ providers including OpenAI, Anthropic, Groq,
|
||||
Azure, AWS Bedrock, Google Vertex AI, and more).
|
||||
|
||||
Features:
|
||||
- Automatic memory injection before LLM calls
|
||||
- Automatic conversation storage after LLM calls
|
||||
- Works with any LiteLLM-supported provider
|
||||
- Zero code changes to existing LiteLLM usage
|
||||
- Multi-user support via separate bank_ids
|
||||
- Document grouping for conversation threading
|
||||
- Direct recall API for manual memory queries
|
||||
- Native client wrappers for OpenAI and Anthropic
|
||||
|
||||
Basic usage:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>>
|
||||
>>> # Configure Hindsight integration
|
||||
>>> configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="user-123", # Use separate bank_ids for multi-user support
|
||||
... store_conversations=True,
|
||||
... inject_memories=True,
|
||||
... )
|
||||
>>>
|
||||
>>> # Enable memory integration
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now use LiteLLM as normal - memory integration is automatic
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "What did we discuss about AI?"}]
|
||||
... )
|
||||
|
||||
Direct recall API:
|
||||
>>> from hindsight_litellm import configure, recall
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>>
|
||||
>>> # Query memories directly
|
||||
>>> memories = recall("what projects am I working on?")
|
||||
>>> for m in memories:
|
||||
... print(f"- [{m.fact_type}] {m.text}")
|
||||
|
||||
Native client wrappers:
|
||||
>>> from openai import OpenAI
|
||||
>>> from hindsight_litellm import wrap_openai
|
||||
>>>
|
||||
>>> client = OpenAI()
|
||||
>>> wrapped = wrap_openai(client, bank_id="user-123")
|
||||
>>>
|
||||
>>> response = wrapped.chat.completions.create(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
|
||||
Works with any LiteLLM-supported provider:
|
||||
>>> # OpenAI
|
||||
>>> litellm.completion(model="gpt-4", messages=[...])
|
||||
>>>
|
||||
>>> # Anthropic
|
||||
>>> litellm.completion(model="claude-3-opus-20240229", messages=[...])
|
||||
>>>
|
||||
>>> # Groq
|
||||
>>> litellm.completion(model="groq/llama-3.1-70b-versatile", messages=[...])
|
||||
>>>
|
||||
>>> # Azure OpenAI
|
||||
>>> litellm.completion(model="azure/gpt-4", messages=[...])
|
||||
>>>
|
||||
>>> # AWS Bedrock
|
||||
>>> litellm.completion(model="bedrock/anthropic.claude-3", messages=[...])
|
||||
>>>
|
||||
>>> # Google Vertex AI
|
||||
>>> litellm.completion(model="vertex_ai/gemini-pro", messages=[...])
|
||||
|
||||
Context manager usage:
|
||||
>>> from hindsight_litellm import hindsight_memory
|
||||
>>>
|
||||
>>> with hindsight_memory(bank_id="user-123"):
|
||||
... response = litellm.completion(model="gpt-4", messages=[...])
|
||||
>>> # Memory integration automatically disabled after context
|
||||
|
||||
Configuration options:
|
||||
- hindsight_api_url: URL of your Hindsight API server
|
||||
- bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
- api_key: Optional API key for Hindsight authentication
|
||||
- store_conversations: Whether to store conversations (default: True)
|
||||
- inject_memories: Whether to inject relevant memories (default: True)
|
||||
- injection_mode: How to inject memories (system_message or prepend_user)
|
||||
- max_memories: Maximum number of memories to inject (None = unlimited)
|
||||
- recall_budget: Budget for memory recall (low, mid, high)
|
||||
- excluded_models: List of model patterns to exclude from interception
|
||||
- verbose: Enable verbose logging
|
||||
- bank_name: Display name for the memory bank
|
||||
- background: Instructions that help Hindsight understand what to remember
|
||||
|
||||
Background example:
|
||||
>>> configure(
|
||||
... bank_id="routing-agent",
|
||||
... background="This agent routes customer requests to support channels. "
|
||||
... "Remember which types of issues should go to which channels.",
|
||||
... )
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, List, Any
|
||||
|
||||
import litellm
|
||||
|
||||
from .config import (
|
||||
configure,
|
||||
get_config,
|
||||
is_configured,
|
||||
reset_config,
|
||||
HindsightConfig,
|
||||
MemoryInjectionMode,
|
||||
)
|
||||
from .callbacks import (
|
||||
HindsightCallback,
|
||||
get_callback,
|
||||
cleanup_callback,
|
||||
)
|
||||
from .wrappers import (
|
||||
recall,
|
||||
arecall,
|
||||
RecallResult,
|
||||
RecallResponse,
|
||||
RecallDebugInfo,
|
||||
reflect,
|
||||
areflect,
|
||||
ReflectResult,
|
||||
ReflectDebugInfo,
|
||||
retain,
|
||||
aretain,
|
||||
RetainResult,
|
||||
RetainDebugInfo,
|
||||
wrap_openai,
|
||||
wrap_anthropic,
|
||||
HindsightOpenAI,
|
||||
HindsightAnthropic,
|
||||
)
|
||||
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
# Track whether we've registered with LiteLLM
|
||||
_enabled = False
|
||||
|
||||
# Store original functions for restoration
|
||||
_original_completion = None
|
||||
_original_acompletion = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class InjectionDebugInfo:
|
||||
"""Debug information from a memory injection operation.
|
||||
|
||||
This is populated when verbose=True in the config and can be retrieved
|
||||
via get_last_injection_debug() after a completion() call.
|
||||
|
||||
Attributes:
|
||||
mode: The injection mode used ("reflect" or "recall")
|
||||
query: The user query used for memory lookup
|
||||
bank_id: The bank ID used
|
||||
memory_context: The formatted memory context that was injected
|
||||
reflect_text: The raw reflect text (when mode="reflect")
|
||||
reflect_facts: The facts used to generate the reflect response (when reflect_include_facts=True)
|
||||
recall_results: The raw recall results (when mode="recall")
|
||||
results_count: Number of memories/results found
|
||||
injected: Whether memories were actually injected into the prompt
|
||||
error: Error message if injection failed (None on success)
|
||||
"""
|
||||
mode: str # "reflect" or "recall"
|
||||
query: str
|
||||
bank_id: str
|
||||
memory_context: str # The formatted context that was injected
|
||||
reflect_text: Optional[str] = None # Raw reflect response text
|
||||
reflect_facts: Optional[List[dict]] = None # Facts used by reflect (when reflect_include_facts=True)
|
||||
recall_results: Optional[List[dict]] = None # Raw recall results
|
||||
results_count: int = 0
|
||||
injected: bool = False
|
||||
error: Optional[str] = None # Error message if injection failed
|
||||
|
||||
|
||||
# Store the last injection debug info (populated when verbose=True)
|
||||
_last_injection_debug: Optional[InjectionDebugInfo] = None
|
||||
|
||||
|
||||
def get_last_injection_debug() -> Optional[InjectionDebugInfo]:
|
||||
"""Get debug info from the last memory injection operation.
|
||||
|
||||
When verbose=True in the config, this returns information about
|
||||
what memories were injected into the last completion() call.
|
||||
|
||||
Returns:
|
||||
InjectionDebugInfo if verbose mode captured injection info, None otherwise
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable, completion, get_last_injection_debug
|
||||
>>> configure(bank_id="my-agent", verbose=True, use_reflect=True)
|
||||
>>> enable()
|
||||
>>> response = completion(model="gpt-4o-mini", messages=[...])
|
||||
>>> debug = get_last_injection_debug()
|
||||
>>> if debug:
|
||||
... print(f"Injected {debug.results_count} memories via {debug.mode}")
|
||||
... print(f"Reflect text: {debug.reflect_text}")
|
||||
"""
|
||||
return _last_injection_debug
|
||||
|
||||
|
||||
def clear_injection_debug() -> None:
|
||||
"""Clear the stored injection debug info."""
|
||||
global _last_injection_debug
|
||||
_last_injection_debug = None
|
||||
|
||||
|
||||
def _inject_memories(messages: List[dict]) -> List[dict]:
|
||||
"""Inject memories into messages list.
|
||||
|
||||
Returns the modified messages list with memories injected into the system message.
|
||||
Uses reflect API when config.use_reflect=True, otherwise uses recall API.
|
||||
|
||||
When verbose=True in config, stores debug info retrievable via get_last_injection_debug().
|
||||
"""
|
||||
global _last_injection_debug
|
||||
import logging
|
||||
|
||||
# Clear previous debug info
|
||||
_last_injection_debug = None
|
||||
|
||||
if not is_configured():
|
||||
return messages
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return messages
|
||||
|
||||
if not messages:
|
||||
return messages
|
||||
|
||||
# Extract user query from last user message
|
||||
user_query = None
|
||||
for msg in reversed(messages):
|
||||
if msg.get("role") == "user":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
user_query = content
|
||||
break
|
||||
|
||||
if not user_query:
|
||||
return messages
|
||||
|
||||
try:
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
# Use bank_id directly (no entity scoping)
|
||||
bank_id = config.bank_id
|
||||
|
||||
# Track debug info
|
||||
mode = "reflect" if config.use_reflect else "recall"
|
||||
reflect_text = None
|
||||
reflect_facts = None
|
||||
recall_results = None
|
||||
results_count = 0
|
||||
memory_context = ""
|
||||
|
||||
# Create client
|
||||
client = Hindsight(base_url=config.hindsight_api_url, timeout=30.0)
|
||||
|
||||
# Use reflect API if use_reflect is enabled
|
||||
if config.use_reflect:
|
||||
# If reflect_include_facts is enabled, use the API directly to include facts
|
||||
if config.reflect_include_facts:
|
||||
from hindsight_client_api.models import reflect_request, reflect_include_options
|
||||
request_obj = reflect_request.ReflectRequest(
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
include=reflect_include_options.ReflectIncludeOptions(facts={}),
|
||||
)
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(client._api.reflect(bank_id, request_obj))
|
||||
# Extract facts from based_on
|
||||
if hasattr(result, 'based_on') and result.based_on:
|
||||
reflect_facts = [
|
||||
{
|
||||
"text": f.text if hasattr(f, 'text') else str(f),
|
||||
"type": getattr(f, 'type', None),
|
||||
"context": getattr(f, 'context', None),
|
||||
}
|
||||
for f in result.based_on
|
||||
]
|
||||
else:
|
||||
result = client.reflect(
|
||||
bank_id=bank_id,
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
)
|
||||
reflect_text = result.text if hasattr(result, 'text') else str(result)
|
||||
|
||||
if not reflect_text:
|
||||
# Store debug info for empty result
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
reflect_text="",
|
||||
reflect_facts=reflect_facts,
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
results_count = 1 # reflect returns a single synthesized response
|
||||
memory_context = (
|
||||
"# Relevant Context from Memory\n"
|
||||
f"{reflect_text}"
|
||||
)
|
||||
else:
|
||||
# Use recall API (original behavior)
|
||||
result = client.recall(
|
||||
bank_id=bank_id,
|
||||
query=user_query,
|
||||
budget=config.recall_budget or "mid",
|
||||
max_tokens=config.max_memory_tokens or 4096,
|
||||
types=config.fact_types,
|
||||
)
|
||||
# client.recall() returns a list directly, not an object with .results
|
||||
if isinstance(result, list):
|
||||
results = result
|
||||
elif hasattr(result, 'results'):
|
||||
results = result.results
|
||||
else:
|
||||
results = []
|
||||
# Convert to dicts for debug info
|
||||
recall_results = [
|
||||
{
|
||||
"text": r.text if hasattr(r, 'text') else str(r),
|
||||
"type": getattr(r, 'type', 'world'),
|
||||
}
|
||||
for r in results
|
||||
]
|
||||
|
||||
if not results:
|
||||
# Store debug info for empty result
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
recall_results=[],
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
# Format memories (apply limit if set, otherwise use all)
|
||||
results_to_use = results[:config.max_memories] if config.max_memories else results
|
||||
memory_lines = []
|
||||
for i, r in enumerate(results_to_use, 1):
|
||||
text = r.text if hasattr(r, 'text') else str(r)
|
||||
fact_type = getattr(r, 'type', 'world')
|
||||
if text:
|
||||
type_label = fact_type.upper() if fact_type else "MEMORY"
|
||||
memory_lines.append(f"{i}. [{type_label}] {text}")
|
||||
|
||||
if not memory_lines:
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context="",
|
||||
recall_results=recall_results,
|
||||
results_count=0,
|
||||
injected=False,
|
||||
)
|
||||
return messages
|
||||
|
||||
results_count = len(memory_lines)
|
||||
memory_context = (
|
||||
"# Relevant Memories\n"
|
||||
"The following information from memory may be relevant:\n\n"
|
||||
+ "\n".join(memory_lines)
|
||||
)
|
||||
|
||||
# Inject into messages
|
||||
updated_messages = list(messages)
|
||||
|
||||
# Find existing system message or create new one
|
||||
found_system = False
|
||||
for i, msg in enumerate(updated_messages):
|
||||
if msg.get("role") == "system":
|
||||
existing_content = msg.get("content", "")
|
||||
updated_messages[i] = {
|
||||
**msg,
|
||||
"content": f"{existing_content}\n\n{memory_context}"
|
||||
}
|
||||
found_system = True
|
||||
break
|
||||
|
||||
if not found_system:
|
||||
updated_messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": memory_context
|
||||
})
|
||||
|
||||
# Store debug info when verbose
|
||||
if config.verbose:
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode=mode,
|
||||
query=user_query,
|
||||
bank_id=bank_id,
|
||||
memory_context=memory_context,
|
||||
reflect_text=reflect_text,
|
||||
reflect_facts=reflect_facts,
|
||||
recall_results=recall_results,
|
||||
results_count=results_count,
|
||||
injected=True,
|
||||
)
|
||||
logger = logging.getLogger("hindsight_litellm")
|
||||
logger.info(f"Injected memories using {mode} into prompt")
|
||||
|
||||
return updated_messages
|
||||
|
||||
except ImportError as e:
|
||||
if config.verbose:
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
f"hindsight_client not installed: {e}. Install with: pip install hindsight-client"
|
||||
)
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode="reflect" if config.use_reflect else "recall",
|
||||
query=user_query or "",
|
||||
bank_id=config.bank_id or "",
|
||||
memory_context="",
|
||||
results_count=0,
|
||||
injected=False,
|
||||
error=f"hindsight_client not installed: {e}",
|
||||
)
|
||||
return messages
|
||||
except Exception as e:
|
||||
# Always set debug info on error when verbose mode is on
|
||||
if config.verbose:
|
||||
logging.getLogger("hindsight_litellm").warning(f"Failed to inject memories: {e}")
|
||||
_last_injection_debug = InjectionDebugInfo(
|
||||
mode="reflect" if config.use_reflect else "recall",
|
||||
query=user_query or "",
|
||||
bank_id=config.bank_id or "",
|
||||
memory_context="",
|
||||
results_count=0,
|
||||
injected=False,
|
||||
error=str(e),
|
||||
)
|
||||
return messages
|
||||
|
||||
|
||||
def _wrapped_completion(*args, **kwargs):
|
||||
"""Wrapper for litellm.completion that injects memories before the call."""
|
||||
# Inject memories into messages
|
||||
if "messages" in kwargs:
|
||||
kwargs["messages"] = _inject_memories(kwargs["messages"])
|
||||
elif args and len(args) > 1:
|
||||
# messages might be second positional arg after model
|
||||
args = list(args)
|
||||
if isinstance(args[1], list):
|
||||
args[1] = _inject_memories(args[1])
|
||||
args = tuple(args)
|
||||
|
||||
# Call original
|
||||
return _original_completion(*args, **kwargs)
|
||||
|
||||
|
||||
async def _wrapped_acompletion(*args, **kwargs):
|
||||
"""Wrapper for litellm.acompletion that injects memories before the call."""
|
||||
# Inject memories into messages
|
||||
if "messages" in kwargs:
|
||||
kwargs["messages"] = _inject_memories(kwargs["messages"])
|
||||
elif args and len(args) > 1:
|
||||
args = list(args)
|
||||
if isinstance(args[1], list):
|
||||
args[1] = _inject_memories(args[1])
|
||||
args = tuple(args)
|
||||
|
||||
# Call original
|
||||
return await _original_acompletion(*args, **kwargs)
|
||||
|
||||
|
||||
def enable() -> None:
|
||||
"""Enable Hindsight memory integration with LiteLLM.
|
||||
|
||||
This monkeypatches LiteLLM functions to:
|
||||
1. Inject relevant memories into prompts before LLM calls
|
||||
2. Store conversations to Hindsight after successful LLM calls
|
||||
|
||||
Must be called after configure() to take effect.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now all LiteLLM calls will have memory integration
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(model="gpt-4", messages=[...])
|
||||
"""
|
||||
global _enabled, _original_completion, _original_acompletion
|
||||
|
||||
if _enabled:
|
||||
return # Already enabled
|
||||
|
||||
if not is_configured():
|
||||
raise RuntimeError(
|
||||
"Hindsight not configured. Call configure() before enable()."
|
||||
)
|
||||
|
||||
# Store original functions and monkeypatch for memory injection
|
||||
_original_completion = litellm.completion
|
||||
_original_acompletion = litellm.acompletion
|
||||
litellm.completion = _wrapped_completion
|
||||
litellm.acompletion = _wrapped_acompletion
|
||||
|
||||
# Get or create the callback instance for storing conversations
|
||||
callback = get_callback()
|
||||
|
||||
# Register callback using litellm.callbacks for conversation storage
|
||||
if callback not in litellm.callbacks:
|
||||
litellm.callbacks.append(callback)
|
||||
|
||||
_enabled = True
|
||||
|
||||
config = get_config()
|
||||
if config and config.verbose:
|
||||
print(f"Hindsight memory enabled for bank: {config.bank_id}")
|
||||
|
||||
|
||||
def disable() -> None:
|
||||
"""Disable Hindsight memory integration with LiteLLM.
|
||||
|
||||
This restores the original LiteLLM functions and removes callbacks,
|
||||
stopping memory injection and conversation storage.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import disable
|
||||
>>> disable() # Stop memory integration
|
||||
"""
|
||||
global _enabled, _original_completion, _original_acompletion
|
||||
|
||||
if not _enabled:
|
||||
return # Already disabled
|
||||
|
||||
# Restore original functions
|
||||
if _original_completion is not None:
|
||||
litellm.completion = _original_completion
|
||||
_original_completion = None
|
||||
if _original_acompletion is not None:
|
||||
litellm.acompletion = _original_acompletion
|
||||
_original_acompletion = None
|
||||
|
||||
# Remove callback from litellm.callbacks
|
||||
callback = get_callback()
|
||||
if callback in litellm.callbacks:
|
||||
litellm.callbacks.remove(callback)
|
||||
|
||||
_enabled = False
|
||||
|
||||
config = get_config()
|
||||
if config and config.verbose:
|
||||
print("Hindsight memory disabled")
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""Check if Hindsight memory integration is currently enabled.
|
||||
|
||||
Returns:
|
||||
True if enable() has been called and not subsequently disabled
|
||||
"""
|
||||
return _enabled
|
||||
|
||||
|
||||
def cleanup() -> None:
|
||||
"""Clean up all Hindsight resources.
|
||||
|
||||
This disables the integration and closes any open connections.
|
||||
Call this when shutting down your application.
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import cleanup
|
||||
>>> cleanup() # Clean up when done
|
||||
"""
|
||||
disable()
|
||||
cleanup_callback()
|
||||
reset_config()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Convenience wrappers - use hindsight_litellm.completion() directly
|
||||
# =============================================================================
|
||||
|
||||
def completion(*args, **kwargs):
|
||||
"""Call LiteLLM completion with Hindsight memory integration.
|
||||
|
||||
This is a convenience wrapper that delegates to litellm.completion().
|
||||
Memory injection and storage happen automatically if configured and enabled.
|
||||
|
||||
Args:
|
||||
*args: Positional arguments passed to litellm.completion()
|
||||
**kwargs: Keyword arguments passed to litellm.completion()
|
||||
|
||||
Returns:
|
||||
LiteLLM ModelResponse object
|
||||
|
||||
Example:
|
||||
>>> import hindsight_litellm
|
||||
>>>
|
||||
>>> hindsight_litellm.configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="my-agent",
|
||||
... )
|
||||
>>> hindsight_litellm.enable()
|
||||
>>>
|
||||
>>> # Use directly - no need to import litellm separately
|
||||
>>> response = hindsight_litellm.completion(
|
||||
... model="gpt-4o-mini",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
"""
|
||||
return litellm.completion(*args, **kwargs)
|
||||
|
||||
|
||||
async def acompletion(*args, **kwargs):
|
||||
"""Call LiteLLM async completion with Hindsight memory integration.
|
||||
|
||||
This is a convenience wrapper that delegates to litellm.acompletion().
|
||||
Memory injection and storage happen automatically if configured and enabled.
|
||||
|
||||
Args:
|
||||
*args: Positional arguments passed to litellm.acompletion()
|
||||
**kwargs: Keyword arguments passed to litellm.acompletion()
|
||||
|
||||
Returns:
|
||||
LiteLLM ModelResponse object
|
||||
|
||||
Example:
|
||||
>>> import hindsight_litellm
|
||||
>>> import asyncio
|
||||
>>>
|
||||
>>> hindsight_litellm.configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="my-agent",
|
||||
... )
|
||||
>>> hindsight_litellm.enable()
|
||||
>>>
|
||||
>>> async def main():
|
||||
... response = await hindsight_litellm.acompletion(
|
||||
... model="gpt-4o-mini",
|
||||
... messages=[{"role": "user", "content": "Hello!"}]
|
||||
... )
|
||||
... return response
|
||||
>>>
|
||||
>>> asyncio.run(main())
|
||||
"""
|
||||
return await litellm.acompletion(*args, **kwargs)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def hindsight_memory(
|
||||
hindsight_api_url: str = "http://localhost:8888",
|
||||
bank_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
store_conversations: bool = True,
|
||||
inject_memories: bool = True,
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
max_memories: Optional[int] = None,
|
||||
max_memory_tokens: int = 4096,
|
||||
recall_budget: str = "mid",
|
||||
fact_types: Optional[List[str]] = None,
|
||||
document_id: Optional[str] = None,
|
||||
excluded_models: Optional[List[str]] = None,
|
||||
verbose: bool = False,
|
||||
bank_name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
):
|
||||
"""Context manager for temporary Hindsight memory integration.
|
||||
|
||||
Use this to enable memory integration for a specific block of code,
|
||||
automatically cleaning up afterwards.
|
||||
|
||||
Args:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations
|
||||
inject_memories: Whether to inject relevant memories
|
||||
injection_mode: How to inject memories
|
||||
max_memories: Maximum number of memories to inject (None = unlimited)
|
||||
max_memory_tokens: Maximum tokens for memory context
|
||||
recall_budget: Budget for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping conversations
|
||||
excluded_models: List of model patterns to exclude
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions for memory extraction
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import hindsight_memory
|
||||
>>> import litellm
|
||||
>>>
|
||||
>>> with hindsight_memory(bank_id="user-123"):
|
||||
... response = litellm.completion(model="gpt-4", messages=[...])
|
||||
>>> # Memory integration automatically disabled after context
|
||||
"""
|
||||
# Save previous state
|
||||
was_enabled = is_enabled()
|
||||
previous_config = get_config()
|
||||
|
||||
try:
|
||||
# Configure and enable
|
||||
configure(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
api_key=api_key,
|
||||
store_conversations=store_conversations,
|
||||
inject_memories=inject_memories,
|
||||
injection_mode=injection_mode,
|
||||
max_memories=max_memories,
|
||||
max_memory_tokens=max_memory_tokens,
|
||||
recall_budget=recall_budget,
|
||||
fact_types=fact_types,
|
||||
document_id=document_id,
|
||||
excluded_models=excluded_models,
|
||||
verbose=verbose,
|
||||
bank_name=bank_name,
|
||||
background=background,
|
||||
)
|
||||
enable()
|
||||
yield
|
||||
finally:
|
||||
# Restore previous state
|
||||
disable()
|
||||
if previous_config:
|
||||
configure(
|
||||
hindsight_api_url=previous_config.hindsight_api_url,
|
||||
bank_id=previous_config.bank_id,
|
||||
api_key=previous_config.api_key,
|
||||
store_conversations=previous_config.store_conversations,
|
||||
inject_memories=previous_config.inject_memories,
|
||||
injection_mode=previous_config.injection_mode,
|
||||
max_memories=previous_config.max_memories,
|
||||
max_memory_tokens=previous_config.max_memory_tokens,
|
||||
recall_budget=previous_config.recall_budget,
|
||||
fact_types=previous_config.fact_types,
|
||||
document_id=previous_config.document_id,
|
||||
excluded_models=previous_config.excluded_models,
|
||||
verbose=previous_config.verbose,
|
||||
bank_name=previous_config.bank_name,
|
||||
background=previous_config.background,
|
||||
)
|
||||
if was_enabled:
|
||||
enable()
|
||||
else:
|
||||
reset_config()
|
||||
|
||||
|
||||
__all__ = [
|
||||
# Main API
|
||||
"configure",
|
||||
"enable",
|
||||
"disable",
|
||||
"is_enabled",
|
||||
"cleanup",
|
||||
"hindsight_memory",
|
||||
# LLM completion wrappers (convenience)
|
||||
"completion",
|
||||
"acompletion",
|
||||
# Direct memory APIs
|
||||
"recall",
|
||||
"arecall",
|
||||
"RecallResult",
|
||||
"reflect",
|
||||
"areflect",
|
||||
"ReflectResult",
|
||||
"retain",
|
||||
"aretain",
|
||||
"RetainResult",
|
||||
# Native client wrappers
|
||||
"wrap_openai",
|
||||
"wrap_anthropic",
|
||||
"HindsightOpenAI",
|
||||
"HindsightAnthropic",
|
||||
# Configuration
|
||||
"get_config",
|
||||
"is_configured",
|
||||
"reset_config",
|
||||
"HindsightConfig",
|
||||
"MemoryInjectionMode",
|
||||
# Injection debug (verbose mode)
|
||||
"get_last_injection_debug",
|
||||
"clear_injection_debug",
|
||||
"InjectionDebugInfo",
|
||||
# Callback (for advanced usage)
|
||||
"HindsightCallback",
|
||||
"get_callback",
|
||||
"cleanup_callback",
|
||||
]
|
||||
640
hindsight-integrations/litellm/hindsight_litellm/callbacks.py
Normal file
640
hindsight-integrations/litellm/hindsight_litellm/callbacks.py
Normal file
|
|
@ -0,0 +1,640 @@
|
|||
"""LiteLLM callback handlers for Hindsight memory integration.
|
||||
|
||||
This module implements LiteLLM's CustomLogger interface to intercept
|
||||
LLM calls and integrate with Hindsight for memory injection and storage.
|
||||
|
||||
Uses direct HTTP calls via requests/httpx to avoid async event loop conflicts
|
||||
when the hindsight_client's async methods are called from LiteLLM callbacks.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import fnmatch
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
import asyncio
|
||||
import threading
|
||||
import concurrent.futures
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from .config import get_config, is_configured, HindsightConfig, MemoryInjectionMode
|
||||
|
||||
# Use requests for sync HTTP calls to avoid async event loop issues
|
||||
try:
|
||||
import requests
|
||||
HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
HAS_REQUESTS = False
|
||||
|
||||
try:
|
||||
import httpx
|
||||
HAS_HTTPX = True
|
||||
except ImportError:
|
||||
HAS_HTTPX = False
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Thread pool for running async operations in background
|
||||
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="hindsight-")
|
||||
|
||||
|
||||
class HindsightCallback(CustomLogger):
|
||||
"""LiteLLM custom logger that integrates with Hindsight memory system.
|
||||
|
||||
This callback handler:
|
||||
1. Injects relevant memories into prompts before LLM calls
|
||||
2. Stores conversations to Hindsight after successful LLM calls
|
||||
|
||||
Features:
|
||||
- Works with 100+ LLM providers via LiteLLM
|
||||
- Deduplication to avoid storing duplicate conversations
|
||||
- Configurable memory injection modes
|
||||
- Support for entity observations in recall
|
||||
|
||||
Usage:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(bank_id="my-agent", hindsight_api_url="http://localhost:8888")
|
||||
>>> enable()
|
||||
>>>
|
||||
>>> # Now all LiteLLM calls will have memory integration
|
||||
>>> import litellm
|
||||
>>> response = litellm.completion(
|
||||
... model="gpt-4",
|
||||
... messages=[{"role": "user", "content": "What did we discuss?"}]
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the Hindsight callback handler."""
|
||||
super().__init__()
|
||||
self._http_session = None
|
||||
self._http_lock = threading.Lock()
|
||||
# Track recently stored conversation hashes for deduplication
|
||||
self._recent_hashes: Set[str] = set()
|
||||
self._max_hash_cache = 1000
|
||||
|
||||
def _get_http_session(self):
|
||||
"""Get or create a requests Session (thread-safe)."""
|
||||
if self._http_session is None:
|
||||
with self._http_lock:
|
||||
if self._http_session is None:
|
||||
if HAS_REQUESTS:
|
||||
self._http_session = requests.Session()
|
||||
elif HAS_HTTPX:
|
||||
self._http_session = httpx.Client(timeout=30.0)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Neither 'requests' nor 'httpx' is installed. "
|
||||
"Please install one: pip install requests"
|
||||
)
|
||||
return self._http_session
|
||||
|
||||
def _http_post(self, url: str, json_data: dict, config: HindsightConfig) -> Optional[dict]:
|
||||
"""Make a synchronous HTTP POST request."""
|
||||
try:
|
||||
session = self._get_http_session()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if config.api_key:
|
||||
headers["Authorization"] = f"Bearer {config.api_key}"
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = session.post(url, json=json_data, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
elif HAS_HTTPX:
|
||||
response = session.post(url, json=json_data, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"HTTP POST failed: {e}")
|
||||
return None
|
||||
|
||||
def _should_skip_model(self, model: str, config: HindsightConfig) -> bool:
|
||||
"""Check if this model should be excluded from interception."""
|
||||
for pattern in config.excluded_models:
|
||||
if fnmatch.fnmatch(model.lower(), pattern.lower()):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _extract_user_query(self, messages: List[Dict[str, Any]]) -> Optional[str]:
|
||||
"""Extract the user's query from the last user message."""
|
||||
for msg in reversed(messages):
|
||||
role = msg.get("role", "")
|
||||
if role == "user":
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
elif isinstance(content, list):
|
||||
# Handle structured content (e.g., vision messages)
|
||||
text_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
text_parts.append(item.get("text", ""))
|
||||
if text_parts:
|
||||
return " ".join(text_parts)
|
||||
return None
|
||||
|
||||
def _compute_conversation_hash(
|
||||
self,
|
||||
user_input: str,
|
||||
assistant_output: str,
|
||||
) -> str:
|
||||
"""Compute a hash for deduplication."""
|
||||
content = f"{user_input.strip().lower()}|{assistant_output.strip().lower()}"
|
||||
return hashlib.md5(content.encode()).hexdigest()[:16]
|
||||
|
||||
def _is_duplicate(self, conv_hash: str) -> bool:
|
||||
"""Check if this conversation was recently stored."""
|
||||
if conv_hash in self._recent_hashes:
|
||||
return True
|
||||
|
||||
# Add to cache, evict oldest if full
|
||||
self._recent_hashes.add(conv_hash)
|
||||
if len(self._recent_hashes) > self._max_hash_cache:
|
||||
# Remove oldest (arbitrary since set, but good enough)
|
||||
self._recent_hashes.pop()
|
||||
|
||||
return False
|
||||
|
||||
def _format_memories(
|
||||
self,
|
||||
results: List[Any],
|
||||
config: HindsightConfig
|
||||
) -> str:
|
||||
"""Format memory recall results into a context string.
|
||||
|
||||
Results can be RecallResult objects (with .text, .type attributes)
|
||||
or dicts (with get() method).
|
||||
"""
|
||||
if not results:
|
||||
return ""
|
||||
|
||||
# Apply limit if set, otherwise use all results
|
||||
results_to_use = results[:config.max_memories] if config.max_memories else results
|
||||
memory_lines = []
|
||||
for i, result in enumerate(results_to_use, 1):
|
||||
# Handle both RecallResult objects and dicts
|
||||
if hasattr(result, 'text'):
|
||||
text = result.text or ""
|
||||
fact_type = getattr(result, 'type', 'world') or "world"
|
||||
weight = getattr(result, 'weight', 0.0) or 0.0
|
||||
else:
|
||||
text = result.get("text", "")
|
||||
fact_type = result.get("type", result.get("fact_type", "world"))
|
||||
weight = result.get("weight", 0.0)
|
||||
|
||||
if text:
|
||||
# Include metadata for context
|
||||
type_label = fact_type.upper() if fact_type else "MEMORY"
|
||||
line = f"{i}. [{type_label}] {text}"
|
||||
if weight > 0 and config.verbose:
|
||||
line += f" (relevance: {weight:.2f})"
|
||||
memory_lines.append(line)
|
||||
|
||||
if not memory_lines:
|
||||
return ""
|
||||
|
||||
return (
|
||||
"# Relevant Memories\n"
|
||||
"The following information from memory may be relevant:\n\n"
|
||||
+ "\n".join(memory_lines)
|
||||
)
|
||||
|
||||
def _inject_memories_into_messages(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
memory_context: str,
|
||||
config: HindsightConfig,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Inject memory context into the messages list."""
|
||||
if not memory_context:
|
||||
return messages
|
||||
|
||||
updated_messages = list(messages) # Make a copy
|
||||
|
||||
if config.injection_mode == MemoryInjectionMode.SYSTEM_MESSAGE:
|
||||
# Find existing system message or create new one
|
||||
for i, msg in enumerate(updated_messages):
|
||||
if msg.get("role") == "system":
|
||||
# Append to existing system message
|
||||
existing_content = msg.get("content", "")
|
||||
updated_messages[i] = {
|
||||
**msg,
|
||||
"content": f"{existing_content}\n\n{memory_context}"
|
||||
}
|
||||
return updated_messages
|
||||
|
||||
# No system message found, prepend one
|
||||
updated_messages.insert(0, {
|
||||
"role": "system",
|
||||
"content": memory_context
|
||||
})
|
||||
|
||||
elif config.injection_mode == MemoryInjectionMode.PREPEND_USER:
|
||||
# Find the last user message and prepend context
|
||||
for i in range(len(updated_messages) - 1, -1, -1):
|
||||
if updated_messages[i].get("role") == "user":
|
||||
original_content = updated_messages[i].get("content", "")
|
||||
if isinstance(original_content, str):
|
||||
updated_messages[i] = {
|
||||
**updated_messages[i],
|
||||
"content": f"{memory_context}\n\n---\n\n{original_content}"
|
||||
}
|
||||
break
|
||||
|
||||
return updated_messages
|
||||
|
||||
def _get_bank_id(self, config: HindsightConfig) -> str:
|
||||
"""Get the bank_id for API calls."""
|
||||
return config.bank_id
|
||||
|
||||
def _recall_memories_sync(
|
||||
self,
|
||||
query: str,
|
||||
config: HindsightConfig
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Recall relevant memories from Hindsight (sync) using direct HTTP."""
|
||||
try:
|
||||
bank_id = self._get_bank_id(config)
|
||||
url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories/recall"
|
||||
|
||||
request_data = {
|
||||
"query": query,
|
||||
"budget": config.recall_budget or "mid",
|
||||
"max_tokens": config.max_memory_tokens or 4096,
|
||||
}
|
||||
if config.fact_types:
|
||||
request_data["types"] = config.fact_types
|
||||
|
||||
response = self._http_post(url, request_data, config)
|
||||
if response and "results" in response:
|
||||
return response["results"]
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to recall memories: {e}")
|
||||
return []
|
||||
|
||||
async def _recall_memories_async(
|
||||
self,
|
||||
query: str,
|
||||
config: HindsightConfig
|
||||
) -> List[Any]:
|
||||
"""Recall relevant memories from Hindsight (async).
|
||||
|
||||
Uses thread pool executor with sync HTTP to avoid event loop conflicts.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
results = await loop.run_in_executor(
|
||||
_executor,
|
||||
self._recall_memories_sync,
|
||||
query,
|
||||
config
|
||||
)
|
||||
|
||||
return results if isinstance(results, list) else []
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to recall memories: {e}")
|
||||
return []
|
||||
|
||||
def _store_conversation_sync(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
config: HindsightConfig,
|
||||
) -> None:
|
||||
"""Store the conversation to Hindsight (sync) using direct HTTP.
|
||||
|
||||
By default, stores the full conversation history passed to the LLM.
|
||||
Each message is stored as a separate item, all linked by document_id.
|
||||
|
||||
Hindsight will process the document as a whole for memory extraction.
|
||||
"""
|
||||
try:
|
||||
# Extract assistant response from the LLM response
|
||||
assistant_output = ""
|
||||
if response.choices and len(response.choices) > 0:
|
||||
choice = response.choices[0]
|
||||
if hasattr(choice, "message") and choice.message:
|
||||
assistant_output = choice.message.content or ""
|
||||
|
||||
if not assistant_output:
|
||||
return
|
||||
|
||||
# Build conversation items - each message becomes a separate item
|
||||
# All linked by document_id for Hindsight to process together
|
||||
items = []
|
||||
for msg in messages:
|
||||
role = msg.get("role", "").upper()
|
||||
content = msg.get("content", "")
|
||||
|
||||
# Skip system messages - they're instructions, not conversation
|
||||
if role == "SYSTEM":
|
||||
continue
|
||||
|
||||
# Skip if this looks like our injected memory context
|
||||
if isinstance(content, str) and content.startswith("# Relevant Memories"):
|
||||
continue
|
||||
|
||||
# Handle structured content (e.g., vision messages)
|
||||
if isinstance(content, list):
|
||||
text_parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
text_parts.append(item.get("text", ""))
|
||||
content = " ".join(text_parts)
|
||||
|
||||
if content:
|
||||
# Map roles to clearer labels
|
||||
label = "USER" if role == "USER" else "ASSISTANT"
|
||||
items.append(f"{label}: {content}")
|
||||
|
||||
# Add the new assistant response
|
||||
items.append(f"ASSISTANT: {assistant_output}")
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
# Use last user message for deduplication hash
|
||||
user_input = self._extract_user_query(messages) or ""
|
||||
|
||||
# Deduplication check
|
||||
conv_hash = self._compute_conversation_hash(user_input, assistant_output)
|
||||
if self._is_duplicate(conv_hash):
|
||||
if config.verbose:
|
||||
logger.debug(f"Skipping duplicate conversation: {conv_hash}")
|
||||
return
|
||||
|
||||
# Build the full conversation as a single item for now
|
||||
# (Future: could store each message as separate item in same document)
|
||||
conversation_text = "\n\n".join(items)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"source": "litellm",
|
||||
"model": model,
|
||||
}
|
||||
|
||||
# Add token usage if available
|
||||
if hasattr(response, "usage") and response.usage:
|
||||
if hasattr(response.usage, "total_tokens"):
|
||||
metadata["tokens"] = str(response.usage.total_tokens)
|
||||
|
||||
bank_id = self._get_bank_id(config)
|
||||
url = f"{config.hindsight_api_url}/v1/default/banks/{bank_id}/memories"
|
||||
|
||||
request_data = {
|
||||
"items": [
|
||||
{
|
||||
"content": conversation_text,
|
||||
"context": f"conversation:litellm:{model}",
|
||||
"metadata": metadata,
|
||||
"document_id": config.document_id, # Group by document
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
self._http_post(url, request_data, config)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Stored conversation to Hindsight bank: {config.bank_id}")
|
||||
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to store conversation: {e}")
|
||||
|
||||
async def _store_conversation_async(
|
||||
self,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: ModelResponse,
|
||||
model: str,
|
||||
config: HindsightConfig,
|
||||
) -> None:
|
||||
"""Store the conversation to Hindsight (async).
|
||||
|
||||
Uses thread pool executor with sync HTTP to avoid event loop conflicts.
|
||||
"""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
await loop.run_in_executor(
|
||||
_executor,
|
||||
self._store_conversation_sync,
|
||||
messages,
|
||||
response,
|
||||
model,
|
||||
config
|
||||
)
|
||||
except Exception as e:
|
||||
if config.verbose:
|
||||
logger.warning(f"Failed to store conversation: {e}")
|
||||
|
||||
# ========== LiteLLM CustomLogger Interface ==========
|
||||
|
||||
def log_pre_api_call(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Called before making the API call (sync).
|
||||
|
||||
This is where we inject memories into the messages.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return
|
||||
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
# Extract user query
|
||||
user_query = self._extract_user_query(messages)
|
||||
if not user_query:
|
||||
return
|
||||
|
||||
# Recall relevant memories
|
||||
memories = self._recall_memories_sync(user_query, config)
|
||||
if not memories:
|
||||
return
|
||||
|
||||
# Format and inject memories
|
||||
memory_context = self._format_memories(memories, config)
|
||||
updated_messages = self._inject_memories_into_messages(
|
||||
messages, memory_context, config
|
||||
)
|
||||
|
||||
# Modify messages list IN-PLACE (don't just reassign kwargs)
|
||||
messages.clear()
|
||||
messages.extend(updated_messages)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Injected {len(memories)} memories into prompt")
|
||||
|
||||
async def async_log_pre_api_call(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
kwargs: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Called before making the API call (async).
|
||||
|
||||
This is where we inject memories into the messages.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.inject_memories:
|
||||
return
|
||||
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
# Extract user query
|
||||
user_query = self._extract_user_query(messages)
|
||||
if not user_query:
|
||||
return
|
||||
|
||||
# Recall relevant memories
|
||||
memories = await self._recall_memories_async(user_query, config)
|
||||
if not memories:
|
||||
return
|
||||
|
||||
# Format and inject memories
|
||||
memory_context = self._format_memories(memories, config)
|
||||
updated_messages = self._inject_memories_into_messages(
|
||||
messages, memory_context, config
|
||||
)
|
||||
|
||||
# Modify messages list IN-PLACE (don't just reassign kwargs)
|
||||
messages.clear()
|
||||
messages.extend(updated_messages)
|
||||
|
||||
if config.verbose:
|
||||
logger.info(f"Injected {len(memories)} memories into prompt")
|
||||
|
||||
def log_success_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after successful API call (sync).
|
||||
|
||||
This is where we store the conversation.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.store_conversations:
|
||||
return
|
||||
|
||||
model = kwargs.get("model", "unknown")
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
return
|
||||
|
||||
# Store the conversation
|
||||
self._store_conversation_sync(messages, response_obj, model, config)
|
||||
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after successful API call (async).
|
||||
|
||||
This is where we store the conversation.
|
||||
"""
|
||||
if not is_configured():
|
||||
return
|
||||
|
||||
config = get_config()
|
||||
if not config or not config.enabled or not config.store_conversations:
|
||||
return
|
||||
|
||||
model = kwargs.get("model", "unknown")
|
||||
if self._should_skip_model(model, config):
|
||||
return
|
||||
|
||||
messages = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
return
|
||||
|
||||
# Store the conversation
|
||||
await self._store_conversation_async(messages, response_obj, model, config)
|
||||
|
||||
def log_failure_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after failed API call (sync)."""
|
||||
# We don't store failed conversations
|
||||
pass
|
||||
|
||||
async def async_log_failure_event(
|
||||
self,
|
||||
kwargs: Dict[str, Any],
|
||||
response_obj: Any,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
) -> None:
|
||||
"""Called after failed API call (async)."""
|
||||
# We don't store failed conversations
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
"""Clean up resources."""
|
||||
with self._http_lock:
|
||||
if self._http_session is not None:
|
||||
try:
|
||||
if HAS_REQUESTS:
|
||||
self._http_session.close()
|
||||
elif HAS_HTTPX:
|
||||
self._http_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._http_session = None
|
||||
self._recent_hashes.clear()
|
||||
|
||||
|
||||
# Global callback instance
|
||||
_callback: Optional[HindsightCallback] = None
|
||||
|
||||
|
||||
def get_callback() -> HindsightCallback:
|
||||
"""Get the global callback instance, creating it if necessary."""
|
||||
global _callback
|
||||
if _callback is None:
|
||||
_callback = HindsightCallback()
|
||||
return _callback
|
||||
|
||||
|
||||
def cleanup_callback() -> None:
|
||||
"""Clean up the global callback instance."""
|
||||
global _callback
|
||||
if _callback is not None:
|
||||
_callback.close()
|
||||
_callback = None
|
||||
232
hindsight-integrations/litellm/hindsight_litellm/config.py
Normal file
232
hindsight-integrations/litellm/hindsight_litellm/config.py
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
"""Global configuration for Hindsight-LiteLLM integration."""
|
||||
|
||||
from typing import Optional, List
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class MemoryInjectionMode(str, Enum):
|
||||
"""How memories should be injected into the prompt."""
|
||||
SYSTEM_MESSAGE = "system_message" # Add as system message
|
||||
PREPEND_USER = "prepend_user" # Prepend to user message
|
||||
DISABLED = "disabled" # Don't inject memories
|
||||
|
||||
|
||||
@dataclass
|
||||
class HindsightConfig:
|
||||
"""Configuration for Hindsight integration with LiteLLM.
|
||||
|
||||
Attributes:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations to Hindsight
|
||||
inject_memories: Whether to inject relevant memories into prompts
|
||||
injection_mode: How to inject memories (system_message or prepend_user)
|
||||
max_memories: Maximum number of memories to inject
|
||||
max_memory_tokens: Maximum tokens for injected memory context
|
||||
recall_budget: Budget level for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter recall (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping stored conversations
|
||||
enabled: Master switch to enable/disable Hindsight integration
|
||||
excluded_models: List of model patterns to exclude from interception
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions for memory extraction
|
||||
use_reflect: Use reflect API instead of recall for memory injection (synthesizes answer)
|
||||
"""
|
||||
|
||||
hindsight_api_url: str = "http://localhost:8888"
|
||||
bank_id: Optional[str] = None
|
||||
api_key: Optional[str] = None
|
||||
store_conversations: bool = True
|
||||
inject_memories: bool = True
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE
|
||||
max_memories: Optional[int] = None # None = no limit (use all results from API)
|
||||
max_memory_tokens: int = 4096
|
||||
recall_budget: str = "mid" # low, mid, high
|
||||
fact_types: Optional[List[str]] = None # world, agent, opinion, observation
|
||||
document_id: Optional[str] = None
|
||||
enabled: bool = True
|
||||
excluded_models: List[str] = field(default_factory=list)
|
||||
verbose: bool = False
|
||||
bank_name: Optional[str] = None # Display name for the memory bank
|
||||
background: Optional[str] = None # Background/instructions for memory extraction
|
||||
use_reflect: bool = False # Use reflect instead of recall for memory injection
|
||||
reflect_include_facts: bool = False # Include facts used by reflect in debug info
|
||||
|
||||
|
||||
# Global configuration instance
|
||||
_global_config: Optional[HindsightConfig] = None
|
||||
|
||||
|
||||
def configure(
|
||||
hindsight_api_url: str = "http://localhost:8888",
|
||||
bank_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
store_conversations: bool = True,
|
||||
inject_memories: bool = True,
|
||||
injection_mode: MemoryInjectionMode = MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
max_memories: Optional[int] = None,
|
||||
max_memory_tokens: int = 4096,
|
||||
recall_budget: str = "mid",
|
||||
fact_types: Optional[List[str]] = None,
|
||||
document_id: Optional[str] = None,
|
||||
enabled: bool = True,
|
||||
excluded_models: Optional[List[str]] = None,
|
||||
verbose: bool = False,
|
||||
bank_name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
use_reflect: bool = False,
|
||||
reflect_include_facts: bool = False,
|
||||
) -> HindsightConfig:
|
||||
"""Configure global Hindsight integration settings for LiteLLM.
|
||||
|
||||
This function sets up the global configuration that will be used by the
|
||||
LiteLLM callbacks to inject memories and store conversations.
|
||||
|
||||
Args:
|
||||
hindsight_api_url: URL of the Hindsight API server
|
||||
bank_id: Memory bank ID for memory operations (required). For multi-user
|
||||
support, use different bank_ids per user (e.g., f"user-{user_id}")
|
||||
api_key: Optional API key for Hindsight authentication
|
||||
store_conversations: Whether to store conversations to Hindsight
|
||||
inject_memories: Whether to inject relevant memories into prompts
|
||||
injection_mode: How to inject memories into the prompt
|
||||
max_memories: Maximum number of memories to inject
|
||||
max_memory_tokens: Maximum tokens for injected memory context
|
||||
recall_budget: Budget level for memory recall (low, mid, high)
|
||||
fact_types: List of fact types to filter (world, agent, opinion, observation)
|
||||
document_id: Optional document ID for grouping stored conversations
|
||||
enabled: Master switch to enable/disable Hindsight integration
|
||||
excluded_models: List of model patterns to exclude from interception
|
||||
verbose: Enable verbose logging
|
||||
bank_name: Optional display name for the memory bank
|
||||
background: Optional background/instructions that help Hindsight understand
|
||||
what information is important to extract and remember from conversations.
|
||||
This is passed to create_bank() to configure the memory bank.
|
||||
use_reflect: Use reflect API instead of recall for memory injection.
|
||||
When True, Hindsight will synthesize a contextual answer based on
|
||||
memories rather than returning raw memory facts.
|
||||
reflect_include_facts: When use_reflect=True, include the facts that
|
||||
were used to generate the reflect response in the debug info.
|
||||
This is useful for debugging what memories the reflect API used.
|
||||
|
||||
Returns:
|
||||
The configured HindsightConfig instance
|
||||
|
||||
Example:
|
||||
>>> from hindsight_litellm import configure, enable
|
||||
>>> configure(
|
||||
... hindsight_api_url="http://localhost:8888",
|
||||
... bank_id="user-123", # Per-user bank for multi-user support
|
||||
... store_conversations=True,
|
||||
... inject_memories=True,
|
||||
... background="This agent routes customer requests to support channels. "
|
||||
... "Remember which types of issues should go to which channels.",
|
||||
... )
|
||||
>>> enable() # Register callbacks with LiteLLM
|
||||
"""
|
||||
global _global_config
|
||||
|
||||
_global_config = HindsightConfig(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
api_key=api_key,
|
||||
store_conversations=store_conversations,
|
||||
inject_memories=inject_memories,
|
||||
injection_mode=injection_mode,
|
||||
max_memories=max_memories,
|
||||
max_memory_tokens=max_memory_tokens,
|
||||
recall_budget=recall_budget,
|
||||
fact_types=fact_types,
|
||||
document_id=document_id,
|
||||
enabled=enabled,
|
||||
excluded_models=excluded_models or [],
|
||||
verbose=verbose,
|
||||
bank_name=bank_name,
|
||||
background=background,
|
||||
use_reflect=use_reflect,
|
||||
reflect_include_facts=reflect_include_facts,
|
||||
)
|
||||
|
||||
# If background or bank_name is provided, create/update the bank
|
||||
if bank_id and (background or bank_name):
|
||||
_create_or_update_bank(
|
||||
hindsight_api_url=hindsight_api_url,
|
||||
bank_id=bank_id,
|
||||
name=bank_name,
|
||||
background=background,
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
return _global_config
|
||||
|
||||
|
||||
def _create_or_update_bank(
|
||||
hindsight_api_url: str,
|
||||
bank_id: str,
|
||||
name: Optional[str] = None,
|
||||
background: Optional[str] = None,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
"""Create or update a memory bank with the given configuration.
|
||||
|
||||
This is called automatically by configure() when background or bank_name is provided.
|
||||
"""
|
||||
try:
|
||||
from hindsight_client import Hindsight
|
||||
|
||||
client = Hindsight(hindsight_api_url)
|
||||
client.create_bank(
|
||||
bank_id=bank_id,
|
||||
name=name,
|
||||
background=background,
|
||||
)
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").info(
|
||||
f"Created/updated bank '{bank_id}' with background"
|
||||
)
|
||||
except ImportError:
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
"hindsight_client not installed. Cannot create bank with background. "
|
||||
"Install with: pip install hindsight-client"
|
||||
)
|
||||
except Exception as e:
|
||||
if verbose:
|
||||
import logging
|
||||
logging.getLogger("hindsight_litellm").warning(
|
||||
f"Failed to create/update bank: {e}"
|
||||
)
|
||||
|
||||
|
||||
def get_config() -> Optional[HindsightConfig]:
|
||||
"""Get the current global configuration.
|
||||
|
||||
Returns:
|
||||
The current HindsightConfig instance, or None if not configured
|
||||
"""
|
||||
return _global_config
|
||||
|
||||
|
||||
def is_configured() -> bool:
|
||||
"""Check if Hindsight has been configured.
|
||||
|
||||
Returns:
|
||||
True if configure() has been called with a valid bank_id
|
||||
"""
|
||||
return (
|
||||
_global_config is not None
|
||||
and _global_config.enabled
|
||||
and _global_config.bank_id is not None
|
||||
)
|
||||
|
||||
|
||||
def reset_config() -> None:
|
||||
"""Reset the global configuration to None."""
|
||||
global _global_config
|
||||
_global_config = None
|
||||
1000
hindsight-integrations/litellm/hindsight_litellm/wrappers.py
Normal file
1000
hindsight-integrations/litellm/hindsight_litellm/wrappers.py
Normal file
File diff suppressed because it is too large
Load diff
59
hindsight-integrations/litellm/pyproject.toml
Normal file
59
hindsight-integrations/litellm/pyproject.toml
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
[project]
|
||||
name = "hindsight-litellm"
|
||||
version = "0.1.0"
|
||||
description = "Universal LLM memory integration via LiteLLM - works with 100+ providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = { text = "MIT" }
|
||||
authors = [
|
||||
{ name = "Vectorize", email = "support@vectorize.io" }
|
||||
]
|
||||
keywords = [
|
||||
"ai",
|
||||
"memory",
|
||||
"llm",
|
||||
"litellm",
|
||||
"openai",
|
||||
"anthropic",
|
||||
"groq",
|
||||
"langchain",
|
||||
"agents",
|
||||
"hindsight",
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"litellm>=1.40.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest-asyncio>=0.21.0",
|
||||
"pytest-mock>=3.10.0",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/vectorize-io/hindsight"
|
||||
Documentation = "https://github.com/vectorize-io/hindsight/tree/main/hindsight-integrations/litellm"
|
||||
Repository = "https://github.com/vectorize-io/hindsight"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["hindsight_litellm"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
1
hindsight-integrations/litellm/tests/__init__.py
Normal file
1
hindsight-integrations/litellm/tests/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
# Tests for hindsight-litellm
|
||||
471
hindsight-integrations/litellm/tests/test_integration.py
Normal file
471
hindsight-integrations/litellm/tests/test_integration.py
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
"""Integration tests for hindsight-litellm."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from hindsight_litellm import (
|
||||
configure,
|
||||
enable,
|
||||
disable,
|
||||
is_enabled,
|
||||
cleanup,
|
||||
get_config,
|
||||
is_configured,
|
||||
reset_config,
|
||||
HindsightConfig,
|
||||
MemoryInjectionMode,
|
||||
)
|
||||
from hindsight_litellm.callbacks import HindsightCallback, get_callback, cleanup_callback
|
||||
|
||||
|
||||
class TestConfiguration:
|
||||
"""Tests for configuration management."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset config before each test."""
|
||||
reset_config()
|
||||
disable()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_configure_creates_config(self):
|
||||
"""Test that configure creates a config object."""
|
||||
config = configure(
|
||||
bank_id="test-agent",
|
||||
hindsight_api_url="http://localhost:8888",
|
||||
)
|
||||
|
||||
assert config is not None
|
||||
assert config.bank_id == "test-agent"
|
||||
assert config.hindsight_api_url == "http://localhost:8888"
|
||||
assert config.enabled is True
|
||||
|
||||
def test_configure_with_all_options(self):
|
||||
"""Test configure with all options."""
|
||||
config = configure(
|
||||
hindsight_api_url="http://custom:9999",
|
||||
bank_id="custom-agent",
|
||||
api_key="secret-key",
|
||||
store_conversations=False,
|
||||
inject_memories=False,
|
||||
injection_mode=MemoryInjectionMode.PREPEND_USER,
|
||||
max_memories=5,
|
||||
max_memory_tokens=1000,
|
||||
recall_budget="high",
|
||||
fact_types=["world", "opinion"],
|
||||
document_id="doc-123",
|
||||
enabled=True,
|
||||
excluded_models=["gpt-3.5*"],
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
assert config.hindsight_api_url == "http://custom:9999"
|
||||
assert config.bank_id == "custom-agent"
|
||||
assert config.api_key == "secret-key"
|
||||
assert config.store_conversations is False
|
||||
assert config.inject_memories is False
|
||||
assert config.injection_mode == MemoryInjectionMode.PREPEND_USER
|
||||
assert config.max_memories == 5
|
||||
assert config.max_memory_tokens == 1000
|
||||
assert config.recall_budget == "high"
|
||||
assert config.fact_types == ["world", "opinion"]
|
||||
assert config.document_id == "doc-123"
|
||||
assert config.excluded_models == ["gpt-3.5*"]
|
||||
assert config.verbose is True
|
||||
|
||||
def test_is_configured_without_bank_id(self):
|
||||
"""Test is_configured returns False without bank_id."""
|
||||
configure(hindsight_api_url="http://localhost:8888")
|
||||
assert is_configured() is False
|
||||
|
||||
def test_is_configured_with_bank_id(self):
|
||||
"""Test is_configured returns True with bank_id."""
|
||||
configure(bank_id="test-agent")
|
||||
assert is_configured() is True
|
||||
|
||||
def test_reset_config(self):
|
||||
"""Test reset_config clears the configuration."""
|
||||
configure(bank_id="test-agent")
|
||||
assert is_configured() is True
|
||||
|
||||
reset_config()
|
||||
assert get_config() is None
|
||||
assert is_configured() is False
|
||||
|
||||
|
||||
class TestEnableDisable:
|
||||
"""Tests for enable/disable functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_enable_without_config_raises(self):
|
||||
"""Test enable raises error without configuration."""
|
||||
with pytest.raises(RuntimeError, match="not configured"):
|
||||
enable()
|
||||
|
||||
def test_enable_registers_callback(self):
|
||||
"""Test enable registers callback with LiteLLM."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
enable()
|
||||
|
||||
callback = get_callback()
|
||||
assert callback in litellm.callbacks
|
||||
assert is_enabled() is True
|
||||
|
||||
def test_disable_removes_callback(self):
|
||||
"""Test disable removes callback from LiteLLM."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
enable()
|
||||
assert is_enabled() is True
|
||||
|
||||
disable()
|
||||
callback = get_callback()
|
||||
assert callback not in litellm.callbacks
|
||||
assert is_enabled() is False
|
||||
|
||||
def test_enable_idempotent(self):
|
||||
"""Test enable is idempotent (can be called multiple times)."""
|
||||
import litellm
|
||||
|
||||
configure(bank_id="test-agent")
|
||||
|
||||
# Enable multiple times
|
||||
enable()
|
||||
enable()
|
||||
enable()
|
||||
|
||||
# Should only have one callback
|
||||
callback = get_callback()
|
||||
assert litellm.callbacks.count(callback) == 1
|
||||
|
||||
|
||||
class TestCallback:
|
||||
"""Tests for the HindsightCallback class."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_extract_user_query_simple(self):
|
||||
"""Test extracting user query from simple messages."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "What is the capital of France?"},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "What is the capital of France?"
|
||||
|
||||
def test_extract_user_query_from_last_user_message(self):
|
||||
"""Test extracting query from last user message."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{"role": "user", "content": "First question"},
|
||||
{"role": "assistant", "content": "First answer"},
|
||||
{"role": "user", "content": "Second question"},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "Second question"
|
||||
|
||||
def test_extract_user_query_structured_content(self):
|
||||
"""Test extracting query from structured content (vision)."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "http://example.com/img.png"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "What's in this image?"
|
||||
|
||||
def test_extract_user_query_multiple_text_parts(self):
|
||||
"""Test extracting query with multiple text parts."""
|
||||
callback = HindsightCallback()
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "First part."},
|
||||
{"type": "text", "text": "Second part."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
query = callback._extract_user_query(messages)
|
||||
assert query == "First part. Second part."
|
||||
|
||||
def test_format_memories(self):
|
||||
"""Test formatting memories into context string."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(bank_id="test", max_memories=10, verbose=False)
|
||||
|
||||
memories = [
|
||||
{"text": "User likes Python", "fact_type": "world", "weight": 0.95},
|
||||
{"text": "User works at Google", "fact_type": "world", "weight": 0.8},
|
||||
]
|
||||
|
||||
formatted = callback._format_memories(memories, config)
|
||||
|
||||
assert "Relevant Memories" in formatted
|
||||
assert "User likes Python" in formatted
|
||||
assert "User works at Google" in formatted
|
||||
assert "[WORLD]" in formatted
|
||||
|
||||
def test_format_memories_with_verbose(self):
|
||||
"""Test formatting memories with verbose mode shows weights."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(bank_id="test", max_memories=10, verbose=True)
|
||||
|
||||
memories = [
|
||||
{"text": "User likes Python", "fact_type": "world", "weight": 0.95},
|
||||
]
|
||||
|
||||
formatted = callback._format_memories(memories, config)
|
||||
|
||||
assert "relevance: 0.95" in formatted
|
||||
|
||||
def test_inject_memories_as_system_message(self):
|
||||
"""Test injecting memories as system message."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["role"] == "system"
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
assert result[1]["role"] == "user"
|
||||
|
||||
def test_inject_memories_prepend_to_existing_system(self):
|
||||
"""Test injecting memories appends to existing system message."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.SYSTEM_MESSAGE,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0]["role"] == "system"
|
||||
assert "You are helpful." in result[0]["content"]
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
|
||||
def test_inject_memories_prepend_user_mode(self):
|
||||
"""Test injecting memories in prepend_user mode."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
injection_mode=MemoryInjectionMode.PREPEND_USER,
|
||||
)
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "What's my name?"},
|
||||
]
|
||||
memory_context = "# Relevant Memories\n1. User is John"
|
||||
|
||||
result = callback._inject_memories_into_messages(messages, memory_context, config)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["role"] == "user"
|
||||
assert "Relevant Memories" in result[0]["content"]
|
||||
assert "What's my name?" in result[0]["content"]
|
||||
|
||||
def test_should_skip_model_exact_match(self):
|
||||
"""Test model exclusion with exact match."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
excluded_models=["gpt-3.5-turbo"],
|
||||
)
|
||||
|
||||
assert callback._should_skip_model("gpt-3.5-turbo", config) is True
|
||||
assert callback._should_skip_model("gpt-4", config) is False
|
||||
|
||||
def test_should_skip_model_wildcard(self):
|
||||
"""Test model exclusion with wildcard pattern."""
|
||||
callback = HindsightCallback()
|
||||
config = HindsightConfig(
|
||||
bank_id="test",
|
||||
excluded_models=["gpt-3.5*", "claude-instant-*"],
|
||||
)
|
||||
|
||||
assert callback._should_skip_model("gpt-3.5-turbo", config) is True
|
||||
assert callback._should_skip_model("gpt-3.5-turbo-16k", config) is True
|
||||
assert callback._should_skip_model("claude-instant-1.2", config) is True
|
||||
assert callback._should_skip_model("gpt-4", config) is False
|
||||
assert callback._should_skip_model("claude-3-opus", config) is False
|
||||
|
||||
|
||||
class TestDeduplication:
|
||||
"""Tests for conversation deduplication."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_compute_conversation_hash(self):
|
||||
"""Test computing conversation hash."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
hash1 = callback._compute_conversation_hash("Hello", "Hi there!")
|
||||
hash2 = callback._compute_conversation_hash("Hello", "Hi there!")
|
||||
hash3 = callback._compute_conversation_hash("Hello", "Different response")
|
||||
|
||||
# Same content should produce same hash
|
||||
assert hash1 == hash2
|
||||
# Different content should produce different hash
|
||||
assert hash1 != hash3
|
||||
|
||||
def test_compute_conversation_hash_case_insensitive(self):
|
||||
"""Test that hash is case insensitive."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
hash1 = callback._compute_conversation_hash("HELLO", "HI THERE!")
|
||||
hash2 = callback._compute_conversation_hash("hello", "hi there!")
|
||||
|
||||
assert hash1 == hash2
|
||||
|
||||
def test_is_duplicate_first_time(self):
|
||||
"""Test first occurrence is not a duplicate."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
result = callback._is_duplicate("abc123")
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_is_duplicate_second_time(self):
|
||||
"""Test second occurrence is a duplicate."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
callback._is_duplicate("abc123") # First time
|
||||
result = callback._is_duplicate("abc123") # Second time
|
||||
|
||||
assert result is True
|
||||
|
||||
def test_is_duplicate_different_hashes(self):
|
||||
"""Test different hashes are not duplicates."""
|
||||
callback = HindsightCallback()
|
||||
|
||||
callback._is_duplicate("abc123")
|
||||
result = callback._is_duplicate("xyz789")
|
||||
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestContextManager:
|
||||
"""Tests for the hindsight_memory context manager."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset state before each test."""
|
||||
cleanup()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_context_manager_enables_and_disables(self):
|
||||
"""Test context manager enables and disables correctly."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
assert is_enabled() is False
|
||||
|
||||
with hindsight_memory(bank_id="test-agent"):
|
||||
assert is_enabled() is True
|
||||
assert get_config().bank_id == "test-agent"
|
||||
|
||||
assert is_enabled() is False
|
||||
|
||||
def test_context_manager_restores_previous_config(self):
|
||||
"""Test context manager restores previous configuration."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
# Set up initial config
|
||||
configure(bank_id="original-agent")
|
||||
enable()
|
||||
assert get_config().bank_id == "original-agent"
|
||||
|
||||
# Use context manager with different config
|
||||
with hindsight_memory(bank_id="temporary-agent"):
|
||||
assert get_config().bank_id == "temporary-agent"
|
||||
|
||||
# Should restore original config
|
||||
assert get_config().bank_id == "original-agent"
|
||||
assert is_enabled() is True
|
||||
|
||||
def test_context_manager_with_fact_types(self):
|
||||
"""Test context manager with fact_types parameter."""
|
||||
from hindsight_litellm import hindsight_memory
|
||||
|
||||
with hindsight_memory(bank_id="test-agent", fact_types=["world", "opinion"]):
|
||||
config = get_config()
|
||||
assert config.fact_types == ["world", "opinion"]
|
||||
|
||||
|
||||
class TestFactTypes:
|
||||
"""Tests for fact_types configuration."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Reset config before each test."""
|
||||
reset_config()
|
||||
|
||||
def teardown_method(self):
|
||||
"""Clean up after each test."""
|
||||
cleanup()
|
||||
|
||||
def test_configure_with_fact_types(self):
|
||||
"""Test configuring with fact_types."""
|
||||
config = configure(
|
||||
bank_id="test-agent",
|
||||
fact_types=["world", "agent", "opinion"],
|
||||
)
|
||||
|
||||
assert config.fact_types == ["world", "agent", "opinion"]
|
||||
|
||||
def test_configure_without_fact_types(self):
|
||||
"""Test configuring without fact_types defaults to None."""
|
||||
config = configure(bank_id="test-agent")
|
||||
|
||||
assert config.fact_types is None
|
||||
Loading…
Reference in a new issue