// Benchmark Visualizer App
let currentBenchmark = null;
let benchmarkData = null;
function selectBenchmark() {
const select = document.getElementById('benchmark-select');
currentBenchmark = select.value;
if (!currentBenchmark) {
document.getElementById('benchmark-content').innerHTML = `
Welcome to Benchmark Visualizer
Select a benchmark from the dropdown above to view results.
`;
return;
}
// Load the selected benchmark
if (currentBenchmark === 'locomo-search') {
loadLocomoResults('search');
} else if (currentBenchmark === 'locomo-think') {
loadLocomoResults('think');
} else if (currentBenchmark === 'longmemeval') {
loadLongMemEvalResults();
}
}
async function loadLocomoResults(mode = 'search') {
try {
const response = await fetch(`/api/locomo?mode=${mode}`);
if (!response.ok) {
const errorData = await response.json();
const modeLabel = mode === 'think' ? 'think' : 'search';
const runCommand = mode === 'think'
? 'uv run python locomo_benchmark.py --use-think'
: 'uv run python locomo_benchmark.py';
document.getElementById('benchmark-content').innerHTML = `
⚠️ Benchmark Results Not Found
${errorData.detail || 'The requested benchmark results are not available.'}
To generate ${modeLabel} mode results:
cd benchmarks/locomo
${runCommand}
Once the benchmark completes, refresh this page and select "${mode === 'think' ? 'LoComo (think)' : 'LoComo (search)'}" again.
`;
return;
}
benchmarkData = await response.json();
console.log(`Loaded locomo data (${mode} mode):`, benchmarkData);
renderLocomoResults(mode);
} catch (e) {
console.error('Error loading benchmark results:', e);
document.getElementById('benchmark-content').innerHTML = `
❌ Error Loading Results
${e.message}
Check the browser console for more details.
`;
}
}
function renderLocomoResults(mode = 'search') {
if (!benchmarkData) return;
const content = document.getElementById('benchmark-content');
try {
// Handle both old and new structure
const results = benchmarkData.item_results || benchmarkData.conversation_results || [];
const numItems = benchmarkData.num_items || results.length;
console.log('Rendering results:', { resultsCount: results.length, numItems });
// Calculate per-category statistics
const categoryStats = {
1: { name: 'Multi-hop', correct: 0, total: 0 },
2: { name: 'Single-hop', correct: 0, total: 0 },
3: { name: 'Temporal', correct: 0, total: 0 },
4: { name: 'Open-domain', correct: 0, total: 0 }
};
// Aggregate across all items
let totalInvalid = 0;
results.forEach(item => {
if (item.metrics && item.metrics.detailed_results) {
item.metrics.detailed_results.forEach(result => {
const category = result.category;
if (categoryStats[category]) {
categoryStats[category].total++;
if (result.is_invalid) {
if (!categoryStats[category].invalid) categoryStats[category].invalid = 0;
categoryStats[category].invalid++;
totalInvalid++;
} else if (result.is_correct) {
categoryStats[category].correct++;
}
}
});
}
});
// Determine title based on mode
const modeLabel = mode === 'think' ? ' (Think Mode)' : ' (Search Mode)';
// Overall stats
const totalInvalidDisplay = totalInvalid > 0
? `
Invalid Questions
${totalInvalid}
`
: '';
const overallHtml = `
LoComo Benchmark${modeLabel} - Overall Performance
${totalInvalid > 0 ? `
⚠️ Note: ${totalInvalid} question(s) marked as invalid due to errors (excluded from accuracy calculation)
` : ''}
Overall Accuracy
${benchmarkData.overall_accuracy.toFixed(2)}%
${totalInvalid > 0 ? `
(${benchmarkData.total_correct} / ${benchmarkData.total_valid || (benchmarkData.total_questions - totalInvalid)})
` : ''}
Correct Answers
${benchmarkData.total_correct} / ${benchmarkData.total_questions}
${totalInvalidDisplay}
Accuracy by Category
${Object.values(categoryStats).map(cat => {
const invalidCount = cat.invalid || 0;
const validTotal = cat.total - invalidCount;
const accuracy = validTotal > 0 ? ((cat.correct / validTotal) * 100).toFixed(1) : 0;
const color = accuracy >= 70 ? '#43a047' : accuracy >= 50 ? '#ff9800' : '#e53935';
const invalidNote = invalidCount > 0 ? `
(${invalidCount} invalid)` : '';
return `
${cat.name}
${accuracy}%
${cat.correct} / ${cat.total}${invalidNote}
`;
}).join('')}
`;
// Filter controls
const filterHtml = `
${totalInvalid > 0 ? '' : ''}
`;
// Build item sections
let itemsHtml = '';
results.forEach((item, idx) => {
const itemId = item.item_id || item.sample_id || `item-${idx}`;
const accuracy = item.metrics.accuracy.toFixed(2);
const correctCount = item.metrics.correct;
const totalCount = item.metrics.total;
itemsHtml += `
📊 ${itemId}
${accuracy}% (${correctCount}/${totalCount})
${renderConversationDetails(item)}
`;
});
content.innerHTML = overallHtml + filterHtml + itemsHtml;
} catch (e) {
console.error('Error rendering Locomo results:', e);
content.innerHTML = `
Error rendering results: ${e.message}
${e.stack}
`;
}
}
function renderConversationDetails(conv) {
if (!conv || !conv.metrics) {
return 'No metrics available
';
}
const results = conv.metrics.detailed_results;
if (!results || !Array.isArray(results) || results.length === 0) {
return 'No detailed results available
';
}
let html = '';
results.forEach((result, idx) => {
const isInvalid = result.is_invalid || false;
const isCorrect = result.is_correct;
const bgColor = isInvalid ? '#fff3cd' : (isCorrect ? '#e8f5e9' : '#ffebee');
const icon = isInvalid ? '⚠️' : (isCorrect ? '✅' : '❌');
const category = getCategoryName(result.category);
html += `
${icon} Question ${idx + 1} ${isInvalid ? 'INVALID' : ''} ${category}
Q: ${result.question}
✓ Correct Answer:
${result.correct_answer}
${isCorrect ? '✓' : '✗'} Predicted Answer:
${result.predicted_answer}
📝 Show Reasoning & Retrieved Memories
${isInvalid ? `
⚠️ Error:
${result.error || 'Question marked as invalid'}
` : ''}
System Reasoning:
${result.reasoning}
Judge Reasoning:
${result.correctness_reasoning || 'N/A'}
Retrieved Memories (${result.retrieved_memories ? result.retrieved_memories.length : 0}):
${renderRetrievedMemories(result.retrieved_memories)}
`;
});
html += '
No memories retrieved
';
}
let html = '';
memories.forEach((mem, idx) => {
if (!mem) return;
const eventDate = mem.event_date ? new Date(mem.event_date).toLocaleString() : 'N/A';
// Determine border color based on fact type
let borderColor = '#42a5f5'; // default blue
let factTypeLabel = '';
if (mem.fact_type) {
factTypeLabel = `
${mem.fact_type.toUpperCase()}`;
if (mem.fact_type === 'world') {
borderColor = '#4caf50'; // green
} else if (mem.fact_type === 'agent') {
borderColor = '#ff9800'; // orange
} else if (mem.fact_type === 'opinion') {
borderColor = '#9c27b0'; // purple
}
}
html += `
Rank #${idx + 1} | Score: ${mem.score ? mem.score.toFixed(4) : 'N/A'} | Event Date: ${eventDate}${factTypeLabel}
${mem.text}
`;
});
html += '
⚠️ Benchmark Results Not Found
${errorData.detail || 'The requested benchmark results are not available.'}
To generate results:
cd benchmarks/longmemeval
uv run python longmemeval_benchmark.py
Once the benchmark completes, refresh this page and select "LongMemEval" again.
`;
return;
}
benchmarkData = await response.json();
console.log('Loaded longmemeval data:', benchmarkData);
renderLongMemEvalResults();
} catch (e) {
console.error('Error loading longmemeval results:', e);
document.getElementById('benchmark-content').innerHTML = `
❌ Error Loading Results
${e.message}
Check the browser console for more details.
`;
}
}
function renderLongMemEvalResults() {
if (!benchmarkData) return;
const content = document.getElementById('benchmark-content');
try {
const results = benchmarkData.item_results || [];
const numItems = benchmarkData.num_items || results.length;
console.log('Rendering longmemeval results:', { resultsCount: results.length, numItems });
// Calculate per-category statistics
const categoryStats = {};
// Aggregate across all items
let totalInvalid = 0;
results.forEach(item => {
if (item.metrics && item.metrics.category_stats) {
Object.entries(item.metrics.category_stats).forEach(([category, stats]) => {
if (!categoryStats[category]) {
categoryStats[category] = { name: category, correct: 0, total: 0, invalid: 0 };
}
categoryStats[category].correct += stats.correct || 0;
categoryStats[category].total += stats.total || 0;
categoryStats[category].invalid += stats.invalid || 0;
});
}
if (item.metrics && item.metrics.detailed_results) {
item.metrics.detailed_results.forEach(result => {
if (result.is_invalid) {
totalInvalid++;
}
});
}
});
// Overall stats
const totalInvalidDisplay = totalInvalid > 0
? `
Invalid Questions
${totalInvalid}
`
: '';
const overallHtml = `
LongMemEval Benchmark - Overall Performance
${totalInvalid > 0 ? `
⚠️ Note: ${totalInvalid} question(s) marked as invalid due to errors (excluded from accuracy calculation)
` : ''}
Overall Accuracy
${benchmarkData.overall_accuracy.toFixed(2)}%
${totalInvalid > 0 ? `
(${benchmarkData.total_correct} / ${benchmarkData.total_valid || (benchmarkData.total_questions - totalInvalid)})
` : ''}
Correct Answers
${benchmarkData.total_correct} / ${benchmarkData.total_questions}
${totalInvalidDisplay}
Accuracy by Category
${Object.values(categoryStats).map(cat => {
const invalidCount = cat.invalid || 0;
const validTotal = cat.total - invalidCount;
const accuracy = validTotal > 0 ? ((cat.correct / validTotal) * 100).toFixed(1) : 0;
const color = accuracy >= 70 ? '#43a047' : accuracy >= 50 ? '#ff9800' : '#e53935';
const invalidNote = invalidCount > 0 ? `
(${invalidCount} invalid)` : '';
return `
${cat.name}
${accuracy}%
${cat.correct} / ${cat.total}${invalidNote}
`;
}).join('')}
`;
// Filter controls
const filterHtml = `
${totalInvalid > 0 ? '' : ''}
`;
// Build item sections
let itemsHtml = '';
results.forEach((item, idx) => {
const itemId = item.item_id || `item-${idx}`;
const accuracy = item.metrics.accuracy.toFixed(2);
const correctCount = item.metrics.correct;
const totalCount = item.metrics.total;
itemsHtml += `
📊 ${itemId}
${accuracy}% (${correctCount}/${totalCount})
${renderLongMemEvalItemDetails(item)}
`;
});
content.innerHTML = overallHtml + filterHtml + itemsHtml;
} catch (e) {
console.error('Error rendering LongMemEval results:', e);
content.innerHTML = `
Error rendering results: ${e.message}
${e.stack}
`;
}
}
function renderLongMemEvalItemDetails(item) {
if (!item || !item.metrics) {
return 'No metrics available
';
}
const results = item.metrics.detailed_results;
if (!results || !Array.isArray(results) || results.length === 0) {
return 'No detailed results available
';
}
let html = '';
results.forEach((result, idx) => {
const isInvalid = result.is_invalid || false;
const isCorrect = result.is_correct;
const bgColor = isInvalid ? '#fff3cd' : (isCorrect ? '#e8f5e9' : '#ffebee');
const icon = isInvalid ? '⚠️' : (isCorrect ? '✅' : '❌');
const category = result.category || 'Unknown';
html += `
${icon} Question ${idx + 1} ${isInvalid ? 'INVALID' : ''} ${category}
Q: ${result.question}
✓ Correct Answer:
${result.correct_answer}
${isCorrect ? '✓' : '✗'} Predicted Answer:
${result.predicted_answer}
📝 Show Reasoning & Retrieved Memories
${isInvalid ? `
⚠️ Error:
${result.error || 'Question marked as invalid'}
` : ''}
System Reasoning:
${result.reasoning || 'N/A'}
Judge Reasoning:
${result.correctness_reasoning || 'N/A'}
Retrieved Memories (${result.retrieved_memories ? result.retrieved_memories.length : 0}):
${renderRetrievedMemories(result.retrieved_memories)}
`;
});
html += '