f89e074dd1
Governed substrate for autonomous agents: scoped identity (passports), audited actions, MCP workspace. Infra IPs and secrets redacted for public release.
436 lines
14 KiB
JavaScript
436 lines
14 KiB
JavaScript
/**
|
|
* Tests for Labs Metrics Service
|
|
* @see ADR-043: Laboratories Feature - MindWorkflow Integration
|
|
*/
|
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
import {
|
|
calculateCost,
|
|
logExecutionMetrics,
|
|
getLabMetricsSummary,
|
|
getNodeMetrics,
|
|
getGlobalMetrics,
|
|
ensureMetricsTable
|
|
} from '../metrics-service.js';
|
|
|
|
// Mock database functions
|
|
vi.mock('../../../database/connection.js', () => ({
|
|
dbGet: vi.fn(),
|
|
dbAll: vi.fn(),
|
|
dbRun: vi.fn(),
|
|
sqlNow: vi.fn(() => 'CURRENT_TIMESTAMP')
|
|
}));
|
|
|
|
// Mock logger
|
|
vi.mock('../../../utils/logger.js', () => ({
|
|
apiLogger: {
|
|
debug: vi.fn(),
|
|
info: vi.fn(),
|
|
error: vi.fn()
|
|
}
|
|
}));
|
|
|
|
import { dbGet, dbAll, dbRun } from '../../../database/connection.js';
|
|
|
|
describe('Labs Metrics Service', () => {
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.resetAllMocks();
|
|
});
|
|
|
|
describe('calculateCost', () => {
|
|
it('should calculate cost for GPT-4o model', () => {
|
|
// GPT-4o: input $0.005/1K, output $0.015/1K
|
|
const cost = calculateCost('gpt-4o', 1000, 500);
|
|
// (1000/1000 * 0.005) + (500/1000 * 0.015) = 0.005 + 0.0075 = 0.0125
|
|
expect(cost).toBe(0.0125);
|
|
});
|
|
|
|
it('should calculate cost for GPT-4o-mini model', () => {
|
|
// GPT-4o-mini: input $0.00015/1K, output $0.0006/1K
|
|
const cost = calculateCost('gpt-4o-mini', 2000, 1000);
|
|
// (2000/1000 * 0.00015) + (1000/1000 * 0.0006) = 0.0003 + 0.0006 = 0.0009
|
|
expect(cost).toBe(0.0009);
|
|
});
|
|
|
|
it('should calculate cost for Claude 3.5 Sonnet model', () => {
|
|
// Claude 3.5 Sonnet: input $0.003/1K, output $0.015/1K
|
|
const cost = calculateCost('claude-3.5-sonnet', 1000, 1000);
|
|
// (1000/1000 * 0.003) + (1000/1000 * 0.015) = 0.003 + 0.015 = 0.018
|
|
expect(cost).toBe(0.018);
|
|
});
|
|
|
|
it('should calculate cost for Claude 3 Opus model', () => {
|
|
// Claude 3 Opus: input $0.015/1K, output $0.075/1K
|
|
const cost = calculateCost('claude-3-opus', 500, 200);
|
|
// (500/1000 * 0.015) + (200/1000 * 0.075) = 0.0075 + 0.015 = 0.0225
|
|
expect(cost).toBe(0.0225);
|
|
});
|
|
|
|
it('should calculate cost for Gemini 2.0 Flash model', () => {
|
|
// Gemini 2.0 Flash: input $0.000075/1K, output $0.0003/1K
|
|
const cost = calculateCost('gemini-2.0-flash', 10000, 5000);
|
|
// (10000/1000 * 0.000075) + (5000/1000 * 0.0003) = 0.00075 + 0.0015 = 0.00225
|
|
expect(cost).toBe(0.00225);
|
|
});
|
|
|
|
it('should use default cost for unknown model', () => {
|
|
// Default: input $0.001/1K, output $0.002/1K
|
|
const cost = calculateCost('unknown-model-xyz', 1000, 1000);
|
|
// (1000/1000 * 0.001) + (1000/1000 * 0.002) = 0.001 + 0.002 = 0.003
|
|
expect(cost).toBe(0.003);
|
|
});
|
|
|
|
it('should handle zero tokens', () => {
|
|
const cost = calculateCost('gpt-4o', 0, 0);
|
|
expect(cost).toBe(0);
|
|
});
|
|
|
|
it('should handle null model gracefully', () => {
|
|
const cost = calculateCost(null, 1000, 1000);
|
|
// Should use default pricing
|
|
expect(cost).toBe(0.003);
|
|
});
|
|
|
|
it('should match model names case-insensitively', () => {
|
|
const cost1 = calculateCost('GPT-4O', 1000, 500);
|
|
const cost2 = calculateCost('gpt-4o', 1000, 500);
|
|
expect(cost1).toBe(cost2);
|
|
});
|
|
|
|
it('should round to 6 decimal places', () => {
|
|
// Use values that would produce many decimal places
|
|
const cost = calculateCost('gpt-4o', 333, 777);
|
|
// Should be rounded to 6 decimal places
|
|
expect(cost.toString().split('.')[1]?.length || 0).toBeLessThanOrEqual(6);
|
|
});
|
|
});
|
|
|
|
describe('logExecutionMetrics', () => {
|
|
it('should log metrics to database', async () => {
|
|
dbRun.mockResolvedValue({ lastInsertRowid: 123 });
|
|
|
|
const metrics = {
|
|
labId: 'lab-001',
|
|
nodeId: 'node-001',
|
|
nodeType: 'ai_agent',
|
|
agentId: 1,
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
inputTokens: 500,
|
|
outputTokens: 200,
|
|
totalTokens: 700,
|
|
executionTime: 1500,
|
|
success: true
|
|
};
|
|
|
|
const result = await logExecutionMetrics(metrics);
|
|
|
|
expect(result).toBe(123);
|
|
expect(dbRun).toHaveBeenCalledTimes(1);
|
|
expect(dbRun).toHaveBeenCalledWith(
|
|
expect.stringContaining('INSERT INTO labs_execution_metrics'),
|
|
expect.arrayContaining(['lab-001', 'node-001', 'ai_agent'])
|
|
);
|
|
});
|
|
|
|
it('should calculate cost if not provided', async () => {
|
|
dbRun.mockResolvedValue({ lastInsertRowid: 124 });
|
|
|
|
const metrics = {
|
|
labId: 'lab-002',
|
|
nodeId: 'node-002',
|
|
nodeType: 'ai_agent',
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
inputTokens: 1000,
|
|
outputTokens: 500
|
|
};
|
|
|
|
await logExecutionMetrics(metrics);
|
|
|
|
// Verify the cost was calculated (10th parameter)
|
|
const callArgs = dbRun.mock.calls[0][1];
|
|
expect(callArgs[9]).toBe(0.0125); // Calculated cost for gpt-4o
|
|
});
|
|
|
|
it('should use provided cost if given', async () => {
|
|
dbRun.mockResolvedValue({ lastInsertRowid: 125 });
|
|
|
|
const metrics = {
|
|
labId: 'lab-003',
|
|
nodeId: 'node-003',
|
|
nodeType: 'ai_agent',
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
inputTokens: 1000,
|
|
outputTokens: 500,
|
|
cost: 0.05 // Custom cost
|
|
};
|
|
|
|
await logExecutionMetrics(metrics);
|
|
|
|
const callArgs = dbRun.mock.calls[0][1];
|
|
expect(callArgs[9]).toBe(0.05); // Should use provided cost
|
|
});
|
|
|
|
it('should handle database errors gracefully', async () => {
|
|
dbRun.mockRejectedValue(new Error('Database error'));
|
|
|
|
const metrics = {
|
|
labId: 'lab-004',
|
|
nodeId: 'node-004',
|
|
nodeType: 'ai_agent',
|
|
model: 'gpt-4o',
|
|
provider: 'openai'
|
|
};
|
|
|
|
const result = await logExecutionMetrics(metrics);
|
|
|
|
expect(result).toBeNull();
|
|
// Should not throw
|
|
});
|
|
|
|
it('should log failed executions', async () => {
|
|
dbRun.mockResolvedValue({ lastInsertRowid: 126 });
|
|
|
|
const metrics = {
|
|
labId: 'lab-005',
|
|
nodeId: 'node-005',
|
|
nodeType: 'ai_agent',
|
|
model: 'gpt-4o',
|
|
provider: 'openai',
|
|
success: false,
|
|
error: 'API rate limit exceeded'
|
|
};
|
|
|
|
await logExecutionMetrics(metrics);
|
|
|
|
const callArgs = dbRun.mock.calls[0][1];
|
|
expect(callArgs[11]).toBe(false); // success
|
|
expect(callArgs[12]).toBe('API rate limit exceeded'); // error
|
|
});
|
|
});
|
|
|
|
describe('getLabMetricsSummary', () => {
|
|
it('should return metrics summary for a lab', async () => {
|
|
dbGet.mockResolvedValue({
|
|
total_executions: 100,
|
|
successful_executions: 95,
|
|
failed_executions: 5,
|
|
total_input_tokens: 50000,
|
|
total_output_tokens: 25000,
|
|
total_tokens: 75000,
|
|
total_cost: 1.5,
|
|
avg_execution_time: 1200,
|
|
min_execution_time: 500,
|
|
max_execution_time: 3000,
|
|
first_execution: '2026-01-01T00:00:00Z',
|
|
last_execution: '2026-01-26T00:00:00Z'
|
|
});
|
|
|
|
dbAll.mockResolvedValueOnce([
|
|
{ model: 'gpt-4o', provider: 'openai', executions: 80, tokens: 60000, cost: 1.2, avg_time: 1100 },
|
|
{ model: 'claude-3.5-sonnet', provider: 'anthropic', executions: 20, tokens: 15000, cost: 0.3, avg_time: 1500 }
|
|
]).mockResolvedValueOnce([
|
|
{ node_type: 'ai_agent', executions: 100, tokens: 75000, cost: 1.5, avg_time: 1200 }
|
|
]);
|
|
|
|
const result = await getLabMetricsSummary('lab-001');
|
|
|
|
expect(result.summary.totalExecutions).toBe(100);
|
|
expect(result.summary.successfulExecutions).toBe(95);
|
|
expect(result.summary.failedExecutions).toBe(5);
|
|
expect(result.summary.successRate).toBe('95.00%');
|
|
expect(result.summary.totalTokens).toBe(75000);
|
|
expect(result.summary.totalCost).toBe(1.5);
|
|
expect(result.byModel).toHaveLength(2);
|
|
expect(result.byNodeType).toHaveLength(1);
|
|
});
|
|
|
|
it('should handle empty results', async () => {
|
|
dbGet.mockResolvedValue(null);
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
const result = await getLabMetricsSummary('lab-empty');
|
|
|
|
expect(result.summary.totalExecutions).toBe(0);
|
|
expect(result.summary.successRate).toBe('0%');
|
|
expect(result.byModel).toEqual([]);
|
|
expect(result.byNodeType).toEqual([]);
|
|
});
|
|
|
|
it('should apply date filters', async () => {
|
|
dbGet.mockResolvedValue({ total_executions: 50 });
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
const startDate = new Date('2026-01-01');
|
|
const endDate = new Date('2026-01-15');
|
|
|
|
await getLabMetricsSummary('lab-001', { startDate, endDate });
|
|
|
|
expect(dbGet).toHaveBeenCalledWith(
|
|
expect.stringContaining('created_at >='),
|
|
expect.arrayContaining(['lab-001', startDate.toISOString(), endDate.toISOString()])
|
|
);
|
|
});
|
|
|
|
it('should handle database errors gracefully', async () => {
|
|
dbGet.mockRejectedValue(new Error('Database error'));
|
|
|
|
const result = await getLabMetricsSummary('lab-error');
|
|
|
|
expect(result.summary.totalExecutions).toBe(0);
|
|
expect(result.byModel).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('getNodeMetrics', () => {
|
|
it('should return metrics for a specific node', async () => {
|
|
dbGet.mockResolvedValue({
|
|
total_executions: 50,
|
|
successful: 48,
|
|
total_tokens: 25000,
|
|
total_cost: 0.5,
|
|
avg_time: 1000
|
|
});
|
|
|
|
dbAll.mockResolvedValue([
|
|
{ id: 1, model: 'gpt-4o', provider: 'openai', input_tokens: 500, output_tokens: 200, total_tokens: 700, cost: 0.01, execution_time_ms: 1000, success: true, error: null, created_at: '2026-01-26T00:00:00Z' }
|
|
]);
|
|
|
|
const result = await getNodeMetrics('node-001');
|
|
|
|
expect(result.summary.totalExecutions).toBe(50);
|
|
expect(result.summary.successfulExecutions).toBe(48);
|
|
expect(result.summary.totalTokens).toBe(25000);
|
|
expect(result.summary.totalCost).toBe(0.5);
|
|
expect(result.recentExecutions).toHaveLength(1);
|
|
});
|
|
|
|
it('should respect limit parameter', async () => {
|
|
dbGet.mockResolvedValue({ total_executions: 100 });
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
await getNodeMetrics('node-001', 50);
|
|
|
|
expect(dbAll).toHaveBeenCalledWith(
|
|
expect.any(String),
|
|
['node-001', 50]
|
|
);
|
|
});
|
|
|
|
it('should handle empty results', async () => {
|
|
dbGet.mockResolvedValue(null);
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
const result = await getNodeMetrics('node-empty');
|
|
|
|
expect(result.summary.totalExecutions).toBe(0);
|
|
expect(result.recentExecutions).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('getGlobalMetrics', () => {
|
|
it('should return global metrics across all labs', async () => {
|
|
dbGet.mockResolvedValue({
|
|
total_executions: 1000,
|
|
unique_labs: 10,
|
|
unique_nodes: 50,
|
|
total_tokens: 500000,
|
|
total_cost: 10.5,
|
|
avg_time: 1100
|
|
});
|
|
|
|
dbAll.mockResolvedValueOnce([
|
|
{ lab_id: 'lab-001', executions: 200, tokens: 100000, cost: 2.0 },
|
|
{ lab_id: 'lab-002', executions: 150, tokens: 75000, cost: 1.5 }
|
|
]).mockResolvedValueOnce([
|
|
{ model: 'gpt-4o', provider: 'openai', executions: 600, tokens: 300000, cost: 6.0 },
|
|
{ model: 'claude-3.5-sonnet', provider: 'anthropic', executions: 400, tokens: 200000, cost: 4.5 }
|
|
]).mockResolvedValueOnce([
|
|
{ date: '2026-01-26', executions: 50, tokens: 25000, cost: 0.5 },
|
|
{ date: '2026-01-25', executions: 45, tokens: 22500, cost: 0.45 }
|
|
]);
|
|
|
|
const result = await getGlobalMetrics();
|
|
|
|
expect(result.summary.totalExecutions).toBe(1000);
|
|
expect(result.summary.uniqueLabs).toBe(10);
|
|
expect(result.summary.uniqueNodes).toBe(50);
|
|
expect(result.summary.totalTokens).toBe(500000);
|
|
expect(result.summary.totalCost).toBe(10.5);
|
|
expect(result.topLabs).toHaveLength(2);
|
|
expect(result.topModels).toHaveLength(2);
|
|
expect(result.dailyTrend).toHaveLength(2);
|
|
});
|
|
|
|
it('should apply date filters', async () => {
|
|
dbGet.mockResolvedValue({ total_executions: 500 });
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
const startDate = new Date('2026-01-01');
|
|
const endDate = new Date('2026-01-15');
|
|
|
|
await getGlobalMetrics({ startDate, endDate });
|
|
|
|
expect(dbGet).toHaveBeenCalledWith(
|
|
expect.stringContaining('created_at >='),
|
|
expect.arrayContaining([startDate.toISOString(), endDate.toISOString()])
|
|
);
|
|
});
|
|
|
|
it('should respect limit parameter', async () => {
|
|
dbGet.mockResolvedValue({ total_executions: 1000 });
|
|
dbAll.mockResolvedValue([]);
|
|
|
|
await getGlobalMetrics({ limit: 5 });
|
|
|
|
expect(dbAll).toHaveBeenCalledWith(
|
|
expect.stringContaining('LIMIT'),
|
|
expect.arrayContaining([5])
|
|
);
|
|
});
|
|
|
|
it('should handle database errors gracefully', async () => {
|
|
dbGet.mockRejectedValue(new Error('Database error'));
|
|
|
|
const result = await getGlobalMetrics();
|
|
|
|
expect(result.summary.totalExecutions).toBe(0);
|
|
expect(result.topLabs).toEqual([]);
|
|
expect(result.topModels).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('ensureMetricsTable', () => {
|
|
it('should create table and indexes', async () => {
|
|
dbRun.mockResolvedValue({});
|
|
|
|
await ensureMetricsTable();
|
|
|
|
expect(dbRun).toHaveBeenCalledTimes(4); // 1 table + 3 indexes
|
|
expect(dbRun).toHaveBeenCalledWith(
|
|
expect.stringContaining('CREATE TABLE IF NOT EXISTS labs_execution_metrics')
|
|
);
|
|
expect(dbRun).toHaveBeenCalledWith(
|
|
expect.stringContaining('CREATE INDEX IF NOT EXISTS idx_labs_metrics_lab_id')
|
|
);
|
|
expect(dbRun).toHaveBeenCalledWith(
|
|
expect.stringContaining('CREATE INDEX IF NOT EXISTS idx_labs_metrics_node_id')
|
|
);
|
|
expect(dbRun).toHaveBeenCalledWith(
|
|
expect.stringContaining('CREATE INDEX IF NOT EXISTS idx_labs_metrics_created_at')
|
|
);
|
|
});
|
|
|
|
it('should handle errors gracefully', async () => {
|
|
dbRun.mockRejectedValue(new Error('Database error'));
|
|
|
|
// Should not throw
|
|
await expect(ensureMetricsTable()).resolves.not.toThrow();
|
|
});
|
|
});
|
|
});
|