@@ -394,6 +402,14 @@ function addDebugPane() {
}
});
+ // Add search input listener for finding nodes
+ document.getElementById(`graph-search-${paneId}`).addEventListener('input', function(e) {
+ const pane = debugPanes.find(p => p.id === paneId);
+ if (pane && pane.cy) {
+ highlightMatchingNodes(paneId, e.target.value);
+ }
+ });
+
debugPanes.push({
id: paneId,
element: paneDiv,
@@ -446,6 +462,7 @@ window.runSearchInPane = async function(paneId) {
const agentId = document.getElementById(`search-agent-${paneId}`).value;
const thinkingBudget = parseInt(document.getElementById(`search-budget-${paneId}`).value);
const topK = parseInt(document.getElementById(`search-top-k-${paneId}`).value);
+ const mmrLambda = parseFloat(document.getElementById(`search-mmr-lambda-${paneId}`).value);
const statusBar = document.getElementById(`debug-status-${paneId}`);
if (!query) {
@@ -465,7 +482,8 @@ window.runSearchInPane = async function(paneId) {
query: query,
agent_id: agentId,
thinking_budget: thinkingBudget,
- top_k: topK
+ top_k: topK,
+ mmr_lambda: mmrLambda
})
});
@@ -536,16 +554,28 @@ function renderResultsTable(paneId, results, trace) {
return;
}
+ // Check if MMR was used
+ const mmrUsed = results.some(r => r.mmr_score !== null && r.mmr_score !== undefined);
+
let html = `
Search Results (${results.length} memories)
-
+
Query: "${trace.query.query_text}"
+ ${mmrUsed ? `
+
+ MMR Diversification Active:
+ 🎯 = Diversified pick (selected for variety) |
+ Orange background = Rank changed by MMR |
+ Orig Rank shows position before MMR reranking
+
+ ` : ''}
| Rank |
+ Orig Rank |
Text |
Context |
Date |
@@ -554,6 +584,9 @@ function renderResultsTable(paneId, results, trace) {
Similarity |
Recency |
Frequency |
+ MMR Score |
+ MMR Rel |
+ MMR Sim |
@@ -570,9 +603,21 @@ function renderResultsTable(paneId, results, trace) {
const recency = visit ? (visit.weights.recency || 0) : 0;
const frequency = visit ? (visit.weights.frequency || 0) : 0;
+ // Get MMR information
+ const originalRank = result.original_rank || (idx + 1);
+ const mmrScore = result.mmr_score;
+ const mmrRelevance = result.mmr_relevance;
+ const mmrMaxSim = result.mmr_max_similarity;
+ const isDiversified = result.mmr_diversified || false;
+
+ // Highlight diversified results
+ const rowBg = isDiversified ? '#fff3e0' : 'white';
+ const rankDisplay = originalRank !== (idx + 1) ? `↑${originalRank}` : originalRank;
+
html += `
-
- | #${idx + 1} |
+
+ | #${idx + 1}${isDiversified ? ' 🎯' : ''} |
+ ${rankDisplay} |
${result.text} |
${result.context || 'N/A'} |
${result.event_date ? new Date(result.event_date).toLocaleDateString() : 'N/A'} |
@@ -581,6 +626,9 @@ function renderResultsTable(paneId, results, trace) {
${similarity.toFixed(4)} |
${recency.toFixed(4)} |
${frequency.toFixed(4)} |
+ ${mmrScore !== null && mmrScore !== undefined ? mmrScore.toFixed(4) : '-'} |
+ ${mmrRelevance !== null && mmrRelevance !== undefined ? mmrRelevance.toFixed(4) : '-'} |
+ ${mmrMaxSim !== null && mmrMaxSim !== undefined ? mmrMaxSim.toFixed(4) : '-'} |
`;
});
@@ -1023,6 +1071,73 @@ function visualizeTrace(paneId, trace) {
}
}
+function highlightMatchingNodes(paneId, searchText) {
+ const pane = debugPanes.find(p => p.id === paneId);
+ if (!pane || !pane.cy) return;
+
+ const countSpan = document.getElementById(`graph-search-count-${paneId}`);
+
+ // If search is empty, reset all styles
+ if (!searchText || searchText.trim() === '') {
+ pane.cy.nodes().style({
+ 'opacity': 1,
+ 'border-width': 2,
+ 'border-color': '#333'
+ });
+ pane.cy.edges().style({
+ 'opacity': 0.8
+ });
+ if (countSpan) countSpan.textContent = '';
+ return;
+ }
+
+ const searchLower = searchText.toLowerCase();
+ let matchCount = 0;
+ const totalNodes = pane.cy.nodes().length;
+
+ // Check each node for matches
+ pane.cy.nodes().forEach(node => {
+ const data = node.data();
+ const text = (data.text || '').toLowerCase();
+ const label = (data.label || '').toLowerCase();
+
+ const matches = text.includes(searchLower) || label.includes(searchLower);
+
+ if (matches) {
+ matchCount++;
+ // Highlight matching nodes - full opacity with thicker orange border
+ node.style({
+ 'opacity': 1,
+ 'border-width': 4,
+ 'border-color': '#ff6f00'
+ });
+ } else {
+ // Dim non-matching nodes
+ node.style({
+ 'opacity': 0.2,
+ 'border-width': 2,
+ 'border-color': '#333'
+ });
+ }
+ });
+
+ // Dim all edges
+ pane.cy.edges().style({
+ 'opacity': 0.2
+ });
+
+ // Update counter
+ if (countSpan) {
+ if (matchCount === 0) {
+ countSpan.textContent = '(no matches)';
+ countSpan.style.color = '#d32f2f';
+ } else {
+ countSpan.textContent = `(${matchCount} of ${totalNodes})`;
+ countSpan.style.color = '#43a047';
+ }
+ }
+}
+
// Table filtering
document.getElementById('table-filter').addEventListener('input', function() {
const filterValue = this.value.toLowerCase();
diff --git a/web/static/js/locomo.js b/web/static/js/locomo.js
index b90b1acc..2b2ef51a 100644
--- a/web/static/js/locomo.js
+++ b/web/static/js/locomo.js
@@ -6,10 +6,13 @@ window.loadLocomoResults = async function() {
try {
const response = await fetch('/api/locomo');
locomoData = await response.json();
+ console.log('Loaded locomo data:', locomoData);
renderLocomoResults();
} catch (e) {
+ console.error('Error loading benchmark results:', e);
document.getElementById('locomo-content').innerHTML = `
- Error loading benchmark results: ${e.message}
+ Error loading benchmark results: ${e.message}
+ Check console for details.
`;
}
}
@@ -19,6 +22,36 @@ function renderLocomoResults() {
const content = document.getElementById('locomo-content');
+ try {
+ // Handle both old and new structure
+ const results = locomoData.item_results || locomoData.conversation_results || [];
+ const numItems = locomoData.num_items || results.length;
+
+ console.log('Rendering results:', { resultsCount: results.length, numItems });
+
+ // Calculate per-category statistics
+ const categoryStats = {
+ 1: { name: 'Multi-hop', correct: 0, total: 0 }, // category 1
+ 2: { name: 'Single-hop', correct: 0, total: 0 }, // category 2
+ 3: { name: 'Temporal', correct: 0, total: 0 }, // category 3
+ 4: { name: 'Open-domain', correct: 0, total: 0 } // category 4
+ };
+
+ // Aggregate across all items
+ results.forEach(item => {
+ if (item.metrics && item.metrics.detailed_results) {
+ item.metrics.detailed_results.forEach(result => {
+ const category = result.category;
+ if (categoryStats[category]) {
+ categoryStats[category].total++;
+ if (result.is_correct) {
+ categoryStats[category].correct++;
+ }
+ }
+ });
+ }
+ });
+
// Overall stats
const overallHtml = `
@@ -33,10 +66,25 @@ function renderLocomoResults() {
${locomoData.total_correct} / ${locomoData.total_questions}
-
Conversations
-
${locomoData.conversation_results.length}
+
Items
+
${numItems}
Accuracy by Category
+
+ ${Object.values(categoryStats).map(cat => {
+ const accuracy = cat.total > 0 ? ((cat.correct / cat.total) * 100).toFixed(1) : 0;
+ const color = accuracy >= 70 ? '#43a047' : accuracy >= 50 ? '#ff9800' : '#e53935';
+ return `
+
+
${cat.name}
+
${accuracy}%
+
${cat.correct} / ${cat.total}
+
+ `;
+ }).join('')}
+
- 📊 ${conv.sample_id}
+ 📊 ${itemId}
${accuracy}% (${correctCount}/${totalCount})
- ${renderConversationDetails(conv)}
+ ${renderConversationDetails(item)}
`;
});
- content.innerHTML = overallHtml + filterHtml + conversationsHtml;
+ content.innerHTML = overallHtml + filterHtml + itemsHtml;
+ } catch (e) {
+ console.error('Error rendering Locomo results:', e);
+ content.innerHTML = `
+
+
Error rendering results: ${e.message}
+
${e.stack}
+
No metrics available
';
+ }
+
const results = conv.metrics.detailed_results;
+ if (!results || !Array.isArray(results) || results.length === 0) {
+ return 'No detailed results available
';
+ }
let html = '';
@@ -150,12 +215,13 @@ function renderConversationDetails(conv) {
}
function renderRetrievedMemories(memories) {
- if (!memories || memories.length === 0) {
+ if (!memories || !Array.isArray(memories) || memories.length === 0) {
return '
No memories retrieved
';
}
let html = '
';
memories.forEach((mem, idx) => {
+ if (!mem) return;
html += `