913 lines
46 KiB
Python
913 lines
46 KiB
Python
"""Benchmark Visualizer - FastHTML App
|
|
|
|
A fast web interface for visualizing benchmark results.
|
|
Supports LoComo and LongMemEval benchmark visualization.
|
|
|
|
Usage:
|
|
python main.py
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fasthtml.common import *
|
|
|
|
# Get the benchmarks directory
|
|
BENCHMARKS_DIR = Path(__file__).resolve().parent.parent
|
|
|
|
# Tailwind + shadcn/ui theme
|
|
def get_head():
|
|
return (
|
|
Script(src="https://cdn.tailwindcss.com"),
|
|
Script("""
|
|
tailwind.config = {
|
|
theme: {
|
|
extend: {
|
|
colors: {
|
|
border: "hsl(214.3 31.8% 91.4%)",
|
|
input: "hsl(214.3 31.8% 91.4%)",
|
|
ring: "hsl(222.2 84% 4.9%)",
|
|
background: "hsl(0 0% 100%)",
|
|
foreground: "hsl(222.2 84% 4.9%)",
|
|
primary: {
|
|
DEFAULT: "hsl(222.2 47.4% 11.2%)",
|
|
foreground: "hsl(210 40% 98%)",
|
|
},
|
|
secondary: {
|
|
DEFAULT: "hsl(210 40% 96.1%)",
|
|
foreground: "hsl(222.2 47.4% 11.2%)",
|
|
},
|
|
destructive: {
|
|
DEFAULT: "hsl(0 84.2% 60.2%)",
|
|
foreground: "hsl(210 40% 98%)",
|
|
},
|
|
muted: {
|
|
DEFAULT: "hsl(210 40% 96.1%)",
|
|
foreground: "hsl(215.4 16.3% 46.9%)",
|
|
},
|
|
accent: {
|
|
DEFAULT: "hsl(210 40% 96.1%)",
|
|
foreground: "hsl(222.2 47.4% 11.2%)",
|
|
},
|
|
success: {
|
|
DEFAULT: "hsl(142.1 76.2% 36.3%)",
|
|
foreground: "hsl(355.7 100% 97.3%)",
|
|
},
|
|
},
|
|
borderRadius: {
|
|
lg: "0.5rem",
|
|
md: "calc(0.5rem - 2px)",
|
|
sm: "calc(0.5rem - 4px)",
|
|
}
|
|
}
|
|
}
|
|
}
|
|
"""),
|
|
Style("""
|
|
@layer base {
|
|
* { border-color: hsl(var(--border)); }
|
|
body {
|
|
background-color: hsl(210 40% 96.1%);
|
|
color: hsl(222.2 84% 4.9%);
|
|
}
|
|
}
|
|
""")
|
|
)
|
|
|
|
# Create FastHTML app
|
|
app, rt = fast_app()
|
|
|
|
|
|
def load_locomo_results(mode: str = "search") -> dict[str, Any] | None:
|
|
"""Load LoComo benchmark results."""
|
|
filename = "benchmark_results_think.json" if mode == "think" else "benchmark_results.json"
|
|
results_path = BENCHMARKS_DIR / "locomo" / "results" / filename
|
|
|
|
if not results_path.exists():
|
|
return None
|
|
|
|
try:
|
|
with open(results_path) as f:
|
|
return json.load(f)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
|
|
|
|
def load_longmemeval_results() -> dict[str, Any] | None:
|
|
"""Load LongMemEval benchmark results."""
|
|
results_path = BENCHMARKS_DIR / "longmemeval" / "results" / "benchmark_results.json"
|
|
|
|
if not results_path.exists():
|
|
return None
|
|
|
|
try:
|
|
with open(results_path) as f:
|
|
return json.load(f)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
|
|
|
|
def get_category_name(category: int | str) -> str:
|
|
"""Map category ID to name for LoComo."""
|
|
if isinstance(category, str):
|
|
return category
|
|
|
|
categories = {
|
|
1: "Multi-hop",
|
|
2: "Single-hop",
|
|
3: "Temporal",
|
|
4: "Open-domain"
|
|
}
|
|
return categories.get(category, "Unknown")
|
|
|
|
|
|
@rt("/")
|
|
def get():
|
|
"""Main page."""
|
|
return (
|
|
Title("Benchmark Visualizer"),
|
|
get_head(),
|
|
Main(
|
|
Div(
|
|
H1("📊 Benchmark Visualizer", cls="text-4xl font-bold text-foreground"),
|
|
P("Analyze and visualize benchmark results", cls="text-muted-foreground mt-2"),
|
|
cls="text-center py-12"
|
|
),
|
|
Div(
|
|
Label("Select a benchmark to view:", cls="block text-sm font-medium text-foreground mb-2"),
|
|
Select(
|
|
Option("-- Choose a benchmark --", value="", selected=True),
|
|
Option("LoComo (search mode)", value="/locomo/search"),
|
|
Option("LoComo (think mode)", value="/locomo/think"),
|
|
Option("LongMemEval", value="/longmemeval"),
|
|
onchange="if(this.value) window.location.href = this.value;",
|
|
cls="w-full px-3 py-2 border border-input rounded-md bg-background text-foreground focus:outline-none focus:ring-2 focus:ring-ring"
|
|
),
|
|
cls="max-w-md mx-auto bg-white border border-border rounded-lg p-6 shadow-sm"
|
|
),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
|
|
|
|
@rt("/locomo/{mode}")
|
|
def get_locomo(mode: str, filter_type: str = "all", category_filter: str = "all"):
|
|
"""Render LoComo results."""
|
|
data = load_locomo_results(mode)
|
|
|
|
if not data:
|
|
mode_label = "think" if mode == "think" else "search"
|
|
extra_args = " --use-think" if mode == "think" else ""
|
|
return (
|
|
Title(f"LoComo ({mode_label}) - Not Found"),
|
|
get_head(),
|
|
Main(
|
|
H1("⚠️ Benchmark Results Not Found", cls="text-3xl font-bold text-foreground mb-4"),
|
|
P(f"The {mode_label} mode results are not available.", cls="text-muted-foreground mb-6"),
|
|
H4("To generate results:", cls="text-lg font-semibold text-foreground mb-2"),
|
|
Pre(f"./scripts/benchmarks/run-locomo.sh --env local{extra_args}", cls="bg-slate-900 text-slate-100 p-4 rounded-md overflow-x-auto text-sm"),
|
|
A("← Back", href="/", cls="inline-flex items-center mt-6 px-4 py-2 bg-primary text-primary-foreground rounded-md hover:bg-primary/90 text-sm font-medium"),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
|
|
all_results = data.get("item_results", data.get("conversation_results", []))
|
|
|
|
# Filter items based on their questions
|
|
results = []
|
|
for item in all_results:
|
|
detailed_results = item.get("metrics", {}).get("detailed_results", [])
|
|
|
|
# Apply correctness filter
|
|
passes_correctness_filter = False
|
|
if filter_type == "all":
|
|
passes_correctness_filter = True
|
|
elif filter_type == "correct":
|
|
# Show items where all questions are correct (and not invalid)
|
|
passes_correctness_filter = detailed_results and all(r.get("is_correct") and not r.get("is_invalid") for r in detailed_results)
|
|
elif filter_type == "incorrect":
|
|
# Show items that have at least one incorrect question
|
|
passes_correctness_filter = any(not r.get("is_correct") and not r.get("is_invalid") for r in detailed_results)
|
|
elif filter_type == "invalid":
|
|
# Show items that have at least one invalid question
|
|
passes_correctness_filter = any(r.get("is_invalid") for r in detailed_results)
|
|
|
|
# Apply category filter
|
|
passes_category_filter = False
|
|
if category_filter == "all":
|
|
passes_category_filter = True
|
|
else:
|
|
# Show items that have at least one question of the specified category
|
|
category_id = int(category_filter)
|
|
passes_category_filter = any(r.get("category") == category_id for r in detailed_results)
|
|
|
|
if passes_correctness_filter and passes_category_filter:
|
|
results.append(item)
|
|
|
|
# Calculate stats (use all_results for overall stats, not filtered results)
|
|
category_stats = {
|
|
1: {"name": "Multi-hop", "correct": 0, "total": 0, "invalid": 0},
|
|
2: {"name": "Single-hop", "correct": 0, "total": 0, "invalid": 0},
|
|
3: {"name": "Temporal", "correct": 0, "total": 0, "invalid": 0},
|
|
4: {"name": "Open-domain", "correct": 0, "total": 0, "invalid": 0}
|
|
}
|
|
|
|
total_invalid = 0
|
|
for item in all_results:
|
|
if item.get("metrics", {}).get("detailed_results"):
|
|
for result in item["metrics"]["detailed_results"]:
|
|
category = result.get("category")
|
|
if category in category_stats:
|
|
category_stats[category]["total"] += 1
|
|
if result.get("is_invalid"):
|
|
category_stats[category]["invalid"] += 1
|
|
total_invalid += 1
|
|
elif result.get("is_correct"):
|
|
category_stats[category]["correct"] += 1
|
|
|
|
mode_label = " (Think Mode)" if mode == "think" else " (Search Mode)"
|
|
|
|
# Overall stats
|
|
stats_html = Div(
|
|
H3(f"LoComo Benchmark{mode_label} - Overall Performance", cls="text-2xl font-bold text-foreground mb-6"),
|
|
Div(
|
|
Div(
|
|
P("Overall Accuracy", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{data['overall_accuracy']:.2f}%", cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
Div(
|
|
P("Correct Answers", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{data['total_correct']} / {data['total_questions']}", cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
Div(
|
|
P("Invalid Questions", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(str(total_invalid), cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
) if total_invalid > 0 else None,
|
|
Div(
|
|
P("Items", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(str(len(all_results)), cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4 mb-8"
|
|
),
|
|
H4("Accuracy by Category", cls="text-xl font-semibold text-foreground mb-4"),
|
|
Div(
|
|
*[
|
|
Div(
|
|
P(cat["name"], cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{(cat['correct'] / (cat['total'] - cat['invalid']) * 100) if (cat['total'] - cat['invalid']) > 0 else 0:.1f}%", cls="text-2xl font-bold text-foreground"),
|
|
P(f"{cat['correct']} / {cat['total']}", cls="text-sm text-muted-foreground mt-1"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
)
|
|
for cat in category_stats.values()
|
|
],
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4"
|
|
)
|
|
)
|
|
|
|
# Filter controls
|
|
filters = Div(
|
|
# Correctness filter
|
|
Div(
|
|
P("Filter by correctness:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All", href=f"/locomo/{mode}?filter_type=all&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("✅ All Correct", href=f"/locomo/{mode}?filter_type=correct&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'correct' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("❌ Has Incorrect", href=f"/locomo/{mode}?filter_type=incorrect&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'incorrect' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("⚠️ Has Invalid", href=f"/locomo/{mode}?filter_type=invalid&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'invalid' else "bg-white text-foreground border border-border hover:bg-accent")) if total_invalid > 0 else None,
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-4"
|
|
),
|
|
# Category filter
|
|
Div(
|
|
P("Filter by question category:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All Categories", href=f"/locomo/{mode}?filter_type={filter_type}&category_filter=all",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Multi-hop", href=f"/locomo/{mode}?filter_type={filter_type}&category_filter=1",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == '1' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Single-hop", href=f"/locomo/{mode}?filter_type={filter_type}&category_filter=2",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == '2' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Temporal", href=f"/locomo/{mode}?filter_type={filter_type}&category_filter=3",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == '3' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Open-domain", href=f"/locomo/{mode}?filter_type={filter_type}&category_filter=4",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == '4' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-4"
|
|
),
|
|
cls="mb-6"
|
|
)
|
|
|
|
# Render items
|
|
items_html = []
|
|
for item in results:
|
|
# Find the original index in all_results
|
|
original_idx = all_results.index(item)
|
|
item_id = item.get("item_id", item.get("sample_id", f"item-{original_idx}"))
|
|
metrics = item.get("metrics", {})
|
|
accuracy = metrics.get("accuracy", 0)
|
|
correct = metrics.get("correct", 0)
|
|
total = metrics.get("total", 0)
|
|
|
|
color = "🟢" if accuracy >= 70 else ("🟡" if accuracy >= 50 else "🔴")
|
|
|
|
# Determine border color based on accuracy
|
|
if accuracy >= 70:
|
|
border_class = "border-l-4 border-green-600"
|
|
bg_class = "hover:bg-green-50"
|
|
elif accuracy >= 50:
|
|
border_class = "border-l-4 border-yellow-500"
|
|
bg_class = "hover:bg-yellow-50"
|
|
else:
|
|
border_class = "border-l-4 border-red-600"
|
|
bg_class = "hover:bg-red-50"
|
|
|
|
# Show preview with link to detail page
|
|
items_html.append(
|
|
A(
|
|
Div(
|
|
Div(
|
|
P(f"{color} {item_id}", cls="text-lg font-semibold text-foreground mb-2"),
|
|
Div(
|
|
Div(
|
|
P("Accuracy", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide"),
|
|
P(f"{accuracy:.1f}%", cls="text-2xl font-bold text-foreground"),
|
|
cls="text-center"
|
|
),
|
|
Div(
|
|
P("Correct", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide"),
|
|
P(f"{correct}/{total}", cls="text-xl font-semibold text-foreground"),
|
|
cls="text-center"
|
|
),
|
|
cls="flex gap-6 items-center"
|
|
),
|
|
cls="p-6"
|
|
),
|
|
cls=f"bg-white border border-border rounded-lg shadow-sm transition-all {border_class} {bg_class}"
|
|
),
|
|
href=f"/locomo/{mode}/item/{original_idx}?filter_type={filter_type}&category_filter={category_filter}",
|
|
cls="block no-underline"
|
|
)
|
|
)
|
|
|
|
return (
|
|
Title(f"LoComo ({mode_label})"),
|
|
get_head(),
|
|
Main(
|
|
Div(
|
|
A("← Back to benchmarks", href="/", cls="inline-flex items-center px-4 py-2 bg-white border border-border rounded-md text-sm font-medium text-foreground hover:bg-accent mb-6"),
|
|
stats_html,
|
|
Hr(cls="my-6 border-border"),
|
|
filters,
|
|
P(f"Showing {len(results)} of {len(all_results)} items", cls="text-sm text-muted-foreground mb-6"),
|
|
Div(
|
|
*items_html,
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4"
|
|
),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
@rt("/locomo/{mode}/item/{item_idx}")
|
|
def get_locomo_item(mode: str, item_idx: int, filter_type: str = "all"):
|
|
"""Render a single LoComo item with questions."""
|
|
data = load_locomo_results(mode)
|
|
if not data:
|
|
return Redirect("/")
|
|
|
|
results = data.get("item_results", data.get("conversation_results", []))
|
|
if item_idx >= len(results):
|
|
return Redirect(f"/locomo/{mode}")
|
|
|
|
item = results[item_idx]
|
|
item_id = item.get("item_id", item.get("sample_id", f"item-{item_idx}"))
|
|
metrics = item.get("metrics", {})
|
|
accuracy = metrics.get("accuracy", 0)
|
|
detailed_results = metrics.get("detailed_results", [])
|
|
|
|
# Filter questions
|
|
filtered_questions = []
|
|
for q_idx, result in enumerate(detailed_results):
|
|
is_invalid = result.get("is_invalid", False)
|
|
is_correct = result.get("is_correct", False)
|
|
|
|
if filter_type == "all":
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "correct" and is_correct and not is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "incorrect" and not is_correct and not is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "invalid" and is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
|
|
# Filters for questions
|
|
has_invalid = any(r.get("is_invalid", False) for r in detailed_results)
|
|
q_filters = Div(
|
|
P("Filter:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All", href=f"/locomo/{mode}/item/{item_idx}?filter_type=all",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("✅ Correct", href=f"/locomo/{mode}/item/{item_idx}?filter_type=correct",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'correct' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("❌ Incorrect", href=f"/locomo/{mode}/item/{item_idx}?filter_type=incorrect",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'incorrect' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("⚠️ Invalid", href=f"/locomo/{mode}/item/{item_idx}?filter_type=invalid",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'invalid' else "bg-white text-foreground border border-border hover:bg-accent")) if has_invalid else None,
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-6"
|
|
)
|
|
|
|
# Render questions
|
|
questions_html = []
|
|
for q_idx, result in filtered_questions:
|
|
is_invalid = result.get("is_invalid", False)
|
|
is_correct = result.get("is_correct", False)
|
|
question = result.get("question", "")
|
|
correct_answer = result.get("correct_answer", "")
|
|
predicted_answer = result.get("predicted_answer", "")
|
|
category = get_category_name(result.get("category", "Unknown"))
|
|
|
|
icon = "⚠️" if is_invalid else ("✅" if is_correct else "❌")
|
|
border_class = "border-l-4 border-yellow-500" if is_invalid else ("border-l-4 border-green-600" if is_correct else "border-l-4 border-red-600")
|
|
|
|
questions_html.append(
|
|
Div(
|
|
# Header
|
|
Div(
|
|
P(f"{icon} Question {q_idx + 1}", cls="text-lg font-semibold text-foreground"),
|
|
P(f"Category: {category}", cls="text-sm text-muted-foreground"),
|
|
cls="mb-4"
|
|
),
|
|
|
|
# Question
|
|
Div(
|
|
P("Question:", cls="text-sm font-medium text-foreground mb-1"),
|
|
P(question, cls="text-foreground"),
|
|
cls="mb-4"
|
|
),
|
|
|
|
# Answers side by side
|
|
Div(
|
|
Div(
|
|
P("✓ Correct Answer", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(correct_answer, cls="bg-green-50 border border-green-200 rounded-md p-3 text-foreground"),
|
|
cls="flex-1"
|
|
),
|
|
Div(
|
|
P(f"{'✓' if is_correct else '✗'} Predicted Answer", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(predicted_answer, cls=f"border rounded-md p-3 text-foreground " + ("bg-green-50 border-green-200" if is_correct else "bg-red-50 border-red-200")),
|
|
cls="flex-1"
|
|
),
|
|
cls="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4"
|
|
),
|
|
|
|
# Details
|
|
Details(
|
|
Summary("📝 Show Reasoning & Retrieved Memories", cls="cursor-pointer font-medium text-foreground hover:text-primary py-2"),
|
|
Div(
|
|
# System Reasoning
|
|
Div(
|
|
P("System Reasoning:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Pre(result.get("reasoning", "N/A"), cls="bg-slate-900 text-slate-100 p-3 rounded-md overflow-x-auto text-sm"),
|
|
cls="mb-4"
|
|
),
|
|
# Judge Reasoning
|
|
Div(
|
|
P("Judge Reasoning:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Pre(result.get("correctness_reasoning", "N/A"), cls="bg-slate-900 text-slate-100 p-3 rounded-md overflow-x-auto text-sm"),
|
|
cls="mb-4"
|
|
),
|
|
# Retrieved Memories
|
|
Div(
|
|
P(f"Retrieved Memories ({len(result.get('retrieved_memories', []))}):", cls="text-sm font-medium text-foreground mb-2"),
|
|
*[
|
|
Div(
|
|
P(f"#{i+1} • Score: {mem.get('score', 0):.4f} • Type: {mem.get('fact_type', 'N/A').upper()}", cls="text-xs text-muted-foreground mb-1"),
|
|
P(mem.get('text', ''), cls="text-sm text-foreground"),
|
|
cls="bg-muted/50 border border-border rounded-md p-3 mb-2"
|
|
)
|
|
for i, mem in enumerate(result.get("retrieved_memories", []))
|
|
] if result.get("retrieved_memories") else [P("No memories retrieved", cls="text-sm text-muted-foreground")],
|
|
),
|
|
cls="mt-3 space-y-2"
|
|
),
|
|
cls="border border-border rounded-md p-4 bg-muted/30"
|
|
),
|
|
|
|
cls=f"bg-white border border-border rounded-lg p-6 mb-4 shadow-sm {border_class}"
|
|
)
|
|
)
|
|
|
|
return (
|
|
Title(f"{item_id}"),
|
|
get_head(),
|
|
Main(
|
|
Div(
|
|
A(f"← Back to LoComo ({mode})", href=f"/locomo/{mode}", cls="inline-flex items-center px-4 py-2 bg-white border border-border rounded-md text-sm font-medium text-foreground hover:bg-accent mb-6"),
|
|
H3(f"📊 {item_id} - {accuracy:.2f}%", cls="text-2xl font-bold text-foreground mb-4"),
|
|
Hr(cls="my-6 border-border"),
|
|
q_filters,
|
|
P(f"Showing {len(filtered_questions)} questions", cls="text-sm text-muted-foreground mb-6"),
|
|
Div(
|
|
*questions_html,
|
|
cls="space-y-4"
|
|
),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
@rt("/longmemeval")
|
|
def get_longmemeval(filter_type: str = "all", category_filter: str = "all"):
|
|
"""Render LongMemEval results."""
|
|
data = load_longmemeval_results()
|
|
|
|
if not data:
|
|
return (
|
|
Title("LongMemEval - Not Found"),
|
|
get_head(),
|
|
Main(
|
|
H1("⚠️ Benchmark Results Not Found"),
|
|
P("The benchmark results are not available."),
|
|
H4("To generate results:"),
|
|
Pre("./scripts/benchmarks/run-longmemeval.sh --env local"),
|
|
A("← Back", href="/", cls="btn mt-3"),
|
|
cls="container"
|
|
)
|
|
)
|
|
|
|
all_results = data.get("item_results", [])
|
|
|
|
# Filter items based on their questions
|
|
results = []
|
|
for item in all_results:
|
|
detailed_results = item.get("metrics", {}).get("detailed_results", [])
|
|
|
|
# Apply correctness filter
|
|
passes_correctness_filter = False
|
|
if filter_type == "all":
|
|
passes_correctness_filter = True
|
|
elif filter_type == "correct":
|
|
# Show items where all questions are correct (and not invalid)
|
|
passes_correctness_filter = detailed_results and all(r.get("is_correct") and not r.get("is_invalid") for r in detailed_results)
|
|
elif filter_type == "incorrect":
|
|
# Show items that have at least one incorrect question
|
|
passes_correctness_filter = any(not r.get("is_correct") and not r.get("is_invalid") for r in detailed_results)
|
|
elif filter_type == "invalid":
|
|
# Show items that have at least one invalid question
|
|
passes_correctness_filter = any(r.get("is_invalid") for r in detailed_results)
|
|
|
|
# Apply category filter
|
|
passes_category_filter = False
|
|
if category_filter == "all":
|
|
passes_category_filter = True
|
|
else:
|
|
# Show items that have at least one question of the specified category
|
|
passes_category_filter = any(r.get("category") == category_filter for r in detailed_results)
|
|
|
|
if passes_correctness_filter and passes_category_filter:
|
|
results.append(item)
|
|
|
|
# Calculate stats (use all_results for overall stats, not filtered results)
|
|
category_stats = {}
|
|
total_invalid = 0
|
|
|
|
for item in all_results:
|
|
if item.get("metrics", {}).get("category_stats"):
|
|
for category, stats in item["metrics"]["category_stats"].items():
|
|
if category not in category_stats:
|
|
category_stats[category] = {"name": category, "correct": 0, "total": 0, "invalid": 0}
|
|
category_stats[category]["correct"] += stats.get("correct", 0)
|
|
category_stats[category]["total"] += stats.get("total", 0)
|
|
category_stats[category]["invalid"] += stats.get("invalid", 0)
|
|
|
|
if item.get("metrics", {}).get("detailed_results"):
|
|
for result in item["metrics"]["detailed_results"]:
|
|
if result.get("is_invalid"):
|
|
total_invalid += 1
|
|
|
|
# Overall stats
|
|
stats_html = Div(
|
|
H3("LongMemEval Benchmark - Overall Performance", cls="text-2xl font-bold text-foreground mb-6"),
|
|
Div(
|
|
Div(
|
|
P("Overall Accuracy", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{data['overall_accuracy']:.2f}%", cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
Div(
|
|
P("Correct Answers", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{data['total_correct']} / {data['total_questions']}", cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
Div(
|
|
P("Invalid Questions", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(str(total_invalid), cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
) if total_invalid > 0 else None,
|
|
Div(
|
|
P("Items", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(str(len(all_results)), cls="text-3xl font-bold text-foreground"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
),
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4 mb-8"
|
|
),
|
|
H4("Accuracy by Category", cls="text-xl font-semibold text-foreground mb-4"),
|
|
Div(
|
|
*[
|
|
Div(
|
|
P(cat["name"], cls="text-xs font-medium text-muted-foreground uppercase tracking-wide mb-2"),
|
|
P(f"{(cat['correct'] / (cat['total'] - cat['invalid']) * 100) if (cat['total'] - cat['invalid']) > 0 else 0:.1f}%", cls="text-2xl font-bold text-foreground"),
|
|
P(f"{cat['correct']} / {cat['total']}", cls="text-sm text-muted-foreground mt-1"),
|
|
cls="bg-white border border-border rounded-lg p-6 text-center shadow-sm"
|
|
)
|
|
for cat in category_stats.values()
|
|
],
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4"
|
|
) if category_stats else None
|
|
)
|
|
|
|
# Filter controls
|
|
filters = Div(
|
|
# Correctness filter
|
|
Div(
|
|
P("Filter by correctness:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All", href=f"/longmemeval?filter_type=all&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("✅ All Correct", href=f"/longmemeval?filter_type=correct&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'correct' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("❌ Has Incorrect", href=f"/longmemeval?filter_type=incorrect&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'incorrect' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("⚠️ Has Invalid", href=f"/longmemeval?filter_type=invalid&category_filter={category_filter}",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'invalid' else "bg-white text-foreground border border-border hover:bg-accent")) if total_invalid > 0 else None,
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-4"
|
|
),
|
|
# Category filter
|
|
Div(
|
|
P("Filter by question category:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All Categories", href=f"/longmemeval?filter_type={filter_type}&category_filter=all",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Multi-session", href=f"/longmemeval?filter_type={filter_type}&category_filter=multi-session",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'multi-session' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Single-session User", href=f"/longmemeval?filter_type={filter_type}&category_filter=single-session-user",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'single-session-user' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Single-session Assistant", href=f"/longmemeval?filter_type={filter_type}&category_filter=single-session-assistant",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'single-session-assistant' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Single-session Preference", href=f"/longmemeval?filter_type={filter_type}&category_filter=single-session-preference",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'single-session-preference' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Temporal Reasoning", href=f"/longmemeval?filter_type={filter_type}&category_filter=temporal-reasoning",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'temporal-reasoning' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("Knowledge Update", href=f"/longmemeval?filter_type={filter_type}&category_filter=knowledge-update",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if category_filter == 'knowledge-update' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-4"
|
|
),
|
|
cls="mb-6"
|
|
)
|
|
|
|
# Render items
|
|
items_html = []
|
|
for item in results:
|
|
# Find the original index in all_results
|
|
original_idx = all_results.index(item)
|
|
item_id = item.get("item_id", f"item-{original_idx}")
|
|
metrics = item.get("metrics", {})
|
|
accuracy = metrics.get("accuracy", 0)
|
|
correct = metrics.get("correct", 0)
|
|
total = metrics.get("total", 0)
|
|
|
|
color = "🟢" if accuracy >= 70 else ("🟡" if accuracy >= 50 else "🔴")
|
|
|
|
# Determine border color based on accuracy
|
|
if accuracy >= 70:
|
|
border_class = "border-l-4 border-green-600"
|
|
bg_class = "hover:bg-green-50"
|
|
elif accuracy >= 50:
|
|
border_class = "border-l-4 border-yellow-500"
|
|
bg_class = "hover:bg-yellow-50"
|
|
else:
|
|
border_class = "border-l-4 border-red-600"
|
|
bg_class = "hover:bg-red-50"
|
|
|
|
items_html.append(
|
|
A(
|
|
Div(
|
|
Div(
|
|
P(f"{color} {item_id}", cls="text-lg font-semibold text-foreground mb-2"),
|
|
Div(
|
|
Div(
|
|
P("Accuracy", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide"),
|
|
P(f"{accuracy:.1f}%", cls="text-2xl font-bold text-foreground"),
|
|
cls="text-center"
|
|
),
|
|
Div(
|
|
P("Correct", cls="text-xs font-medium text-muted-foreground uppercase tracking-wide"),
|
|
P(f"{correct}/{total}", cls="text-xl font-semibold text-foreground"),
|
|
cls="text-center"
|
|
),
|
|
cls="flex gap-6 items-center"
|
|
),
|
|
cls="p-6"
|
|
),
|
|
cls=f"bg-white border border-border rounded-lg shadow-sm transition-all {border_class} {bg_class}"
|
|
),
|
|
href=f"/longmemeval/item/{original_idx}?filter_type={filter_type}&category_filter={category_filter}",
|
|
cls="block no-underline"
|
|
)
|
|
)
|
|
|
|
return (
|
|
Title("LongMemEval"),
|
|
get_head(),
|
|
Main(
|
|
Div(
|
|
A("← Back to benchmarks", href="/", cls="inline-flex items-center px-4 py-2 bg-white border border-border rounded-md text-sm font-medium text-foreground hover:bg-accent mb-6"),
|
|
stats_html,
|
|
Hr(cls="my-6 border-border"),
|
|
filters,
|
|
P(f"Showing {len(results)} of {len(all_results)} items", cls="text-sm text-muted-foreground mb-6"),
|
|
Div(
|
|
*items_html,
|
|
cls="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4"
|
|
),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
@rt("/longmemeval/item/{item_idx}")
|
|
def get_longmemeval_item(item_idx: int, filter_type: str = "all"):
|
|
"""Render a single LongMemEval item with questions."""
|
|
data = load_longmemeval_results()
|
|
if not data:
|
|
return Redirect("/")
|
|
|
|
results = data.get("item_results", [])
|
|
if item_idx >= len(results):
|
|
return Redirect("/longmemeval")
|
|
|
|
item = results[item_idx]
|
|
item_id = item.get("item_id", f"item-{item_idx}")
|
|
metrics = item.get("metrics", {})
|
|
accuracy = metrics.get("accuracy", 0)
|
|
detailed_results = metrics.get("detailed_results", [])
|
|
|
|
# Filter questions
|
|
filtered_questions = []
|
|
for q_idx, result in enumerate(detailed_results):
|
|
is_invalid = result.get("is_invalid", False)
|
|
is_correct = result.get("is_correct", False)
|
|
|
|
if filter_type == "all":
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "correct" and is_correct and not is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "incorrect" and not is_correct and not is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
elif filter_type == "invalid" and is_invalid:
|
|
filtered_questions.append((q_idx, result))
|
|
|
|
# Filters for questions
|
|
has_invalid = any(r.get("is_invalid", False) for r in detailed_results)
|
|
q_filters = Div(
|
|
P("Filter:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(
|
|
A("All", href=f"/longmemeval/item/{item_idx}?filter_type=all",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'all' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("✅ Correct", href=f"/longmemeval/item/{item_idx}?filter_type=correct",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'correct' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("❌ Incorrect", href=f"/longmemeval/item/{item_idx}?filter_type=incorrect",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'incorrect' else "bg-white text-foreground border border-border hover:bg-accent")),
|
|
A("⚠️ Invalid", href=f"/longmemeval/item/{item_idx}?filter_type=invalid",
|
|
cls="px-3 py-1.5 rounded-md text-sm font-medium " + ("bg-primary text-primary-foreground" if filter_type == 'invalid' else "bg-white text-foreground border border-border hover:bg-accent")) if has_invalid else None,
|
|
cls="flex flex-wrap gap-2"
|
|
),
|
|
cls="mb-6"
|
|
)
|
|
|
|
# Render questions
|
|
questions_html = []
|
|
for q_idx, result in filtered_questions:
|
|
is_invalid = result.get("is_invalid", False)
|
|
is_correct = result.get("is_correct", False)
|
|
question = result.get("question", "")
|
|
correct_answer = result.get("correct_answer", "")
|
|
predicted_answer = result.get("predicted_answer", "")
|
|
category = result.get("category", "Unknown")
|
|
|
|
icon = "⚠️" if is_invalid else ("✅" if is_correct else "❌")
|
|
border_class = "border-l-4 border-yellow-500" if is_invalid else ("border-l-4 border-green-600" if is_correct else "border-l-4 border-red-600")
|
|
|
|
questions_html.append(
|
|
Div(
|
|
# Header
|
|
Div(
|
|
P(f"{icon} Question {q_idx + 1}", cls="text-lg font-semibold text-foreground"),
|
|
P(f"Category: {category}", cls="text-sm text-muted-foreground"),
|
|
cls="mb-4"
|
|
),
|
|
|
|
# Question
|
|
Div(
|
|
P("Question:", cls="text-sm font-medium text-foreground mb-1"),
|
|
P(question, cls="text-foreground"),
|
|
cls="mb-4"
|
|
),
|
|
|
|
# Answers side by side
|
|
Div(
|
|
Div(
|
|
P("✓ Correct Answer", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(correct_answer, cls="bg-green-50 border border-green-200 rounded-md p-3 text-foreground"),
|
|
cls="flex-1"
|
|
),
|
|
Div(
|
|
P(f"{'✓' if is_correct else '✗'} Predicted Answer", cls="text-sm font-medium text-foreground mb-2"),
|
|
Div(predicted_answer, cls=f"border rounded-md p-3 text-foreground " + ("bg-green-50 border-green-200" if is_correct else "bg-red-50 border-red-200")),
|
|
cls="flex-1"
|
|
),
|
|
cls="grid grid-cols-1 md:grid-cols-2 gap-4 mb-4"
|
|
),
|
|
|
|
# Details
|
|
Details(
|
|
Summary("📝 Show Reasoning & Retrieved Memories", cls="cursor-pointer font-medium text-foreground hover:text-primary py-2"),
|
|
Div(
|
|
# System Reasoning
|
|
Div(
|
|
P("System Reasoning:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Pre(result.get("reasoning", "N/A"), cls="bg-slate-900 text-slate-100 p-3 rounded-md overflow-x-auto text-sm"),
|
|
cls="mb-4"
|
|
),
|
|
# Judge Reasoning
|
|
Div(
|
|
P("Judge Reasoning:", cls="text-sm font-medium text-foreground mb-2"),
|
|
Pre(result.get("correctness_reasoning", "N/A"), cls="bg-slate-900 text-slate-100 p-3 rounded-md overflow-x-auto text-sm"),
|
|
cls="mb-4"
|
|
),
|
|
# Retrieved Memories
|
|
Div(
|
|
P(f"Retrieved Memories ({len(result.get('retrieved_memories', []))}):", cls="text-sm font-medium text-foreground mb-2"),
|
|
*[
|
|
Div(
|
|
P(f"#{i+1} • Score: {mem.get('score', 0):.4f} • Type: {mem.get('fact_type', 'N/A').upper()}", cls="text-xs text-muted-foreground mb-1"),
|
|
P(mem.get('text', ''), cls="text-sm text-foreground"),
|
|
cls="bg-muted/50 border border-border rounded-md p-3 mb-2"
|
|
)
|
|
for i, mem in enumerate(result.get("retrieved_memories", []))
|
|
] if result.get("retrieved_memories") else [P("No memories retrieved", cls="text-sm text-muted-foreground")],
|
|
),
|
|
cls="mt-3 space-y-2"
|
|
),
|
|
cls="border border-border rounded-md p-4 bg-muted/30"
|
|
),
|
|
|
|
cls=f"bg-white border border-border rounded-lg p-6 mb-4 shadow-sm {border_class}"
|
|
)
|
|
)
|
|
|
|
return (
|
|
Title(f"{item_id}"),
|
|
get_head(),
|
|
Main(
|
|
Div(
|
|
A("← Back to LongMemEval", href="/longmemeval", cls="inline-flex items-center px-4 py-2 bg-white border border-border rounded-md text-sm font-medium text-foreground hover:bg-accent mb-6"),
|
|
H3(f"📊 {item_id} - {accuracy:.2f}%", cls="text-2xl font-bold text-foreground mb-4"),
|
|
Hr(cls="my-6 border-border"),
|
|
q_filters,
|
|
P(f"Showing {len(filtered_questions)} questions", cls="text-sm text-muted-foreground mb-6"),
|
|
Div(
|
|
*questions_html,
|
|
cls="space-y-4"
|
|
),
|
|
cls="container mx-auto max-w-7xl px-4 py-8"
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
print("🚀 Starting Benchmark Visualizer...")
|
|
print("📊 Server running at: http://localhost:8001")
|
|
uvicorn.run("main:app", host="127.0.0.1", port=8001, reload=True)
|