15ea23d5d6
* feat: introduce hindsight-api-slim and hindsight-all-slim packages Closes #552 - Move all source code from hindsight-api/ to new hindsight-api-slim/ - hindsight-api-slim has heavy ML deps (torch, sentence-transformers, transformers, einops, flashrank, mlx, mlx-lm, safetensors) and pg0-embedded as optional extras: [local-ml], [embedded-db], [all] - hindsight-api becomes a zero-code meta-package depending on hindsight-api-slim[all] for full backward compatibility - Add hindsight-all-slim meta-package: hindsight-api-slim + client + embed - hindsight-all updated to depend on hindsight-api-slim[all] - pg0.py: lazy-import pg0 with clear ImportError pointing to [embedded-db] - Dockerfile: replace sed hack with proper uv sync --extra flags - Update release.yml, test.yml, lint.sh, release.sh, CLAUDE.md and all path references throughout the repo * refactor: rename hindsight/ directory to hindsight-all/ * docs: document hindsight-api-slim and hindsight-all-slim package variants Add package variants table and extras explanation to installation.md * docs: remove emojis from installation.md, use professional tone * docs: link Docker slim variant to pip package variants section * docs: consolidate Docker image variants into single table * ci: fix working-directory paths after package restructure - Replace all hindsight-api → hindsight-api-slim in test.yml - Replace hindsight → hindsight-all in test.yml - Add --extra embedded-db to test-embed API install step * ci: add local-ml and embedded-db extras to API sync steps These extras were previously implicit in the old hindsight-api package (which bundled everything). Now that hindsight-api-slim uses optional extras, we must explicitly request local-ml and embedded-db in CI. * ci: add API install step with embedded-db to test-embed smoke test The smoke test starts hindsight-api as a daemon, which requires pg0-embedded. Add a dedicated install step for hindsight-api-slim with embedded-db extra so the daemon can start successfully. * ci: remove --no-install-project when using optional extras When --no-install-project is combined with --extra, the optional deps are not installed because extras require the project to be active. Remove --no-install-project from steps that need local-ml or embedded-db. * ci: fix ordering of uv sync steps to preserve optional extras When uv sync runs for a different workspace member, it removes optional extras installed for other members. Fix by always running extra-requiring API sync last, after other workspace member syncs. Also remove --no-install-project from embedded-db sync in test-embed, as --no-install-project prevents optional extras from being active. * ci: add local-ml extra to test-embed API install for smoke test The smoke test starts the full API server which needs sentence-transformers for local embeddings (default provider). Add local-ml extra to the install. * ci: simplify extras with --all-extras and add slim pip smoke test - Replace explicit --extra local-ml --extra embedded-db with --all-extras for cleaner, more maintainable sync steps - Add test-pip-slim job: tests hindsight-api-slim[embedded-db] without local ML models, using Cohere for embeddings/reranking (mirrors Docker slim smoke test approach) * ci: simplify slim smoke test to health check only (mirrors Docker test)
408 lines
13 KiB
Python
408 lines
13 KiB
Python
"""
|
|
Unit tests for OpenTelemetry tracing instrumentation.
|
|
|
|
Tests the tracing module's ability to record LLM calls with GenAI semantic conventions.
|
|
"""
|
|
|
|
import json
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from hindsight_api.tracing import (
|
|
PROVIDER_NAME_MAPPING,
|
|
GenAIAttributes,
|
|
LLMSpanRecorder,
|
|
NoOpLLMSpanRecorder,
|
|
_truncate_content,
|
|
create_operation_span,
|
|
initialize_tracing,
|
|
is_tracing_enabled,
|
|
)
|
|
|
|
|
|
def test_provider_name_mapping():
|
|
"""Test that provider names are correctly mapped to GenAI conventions."""
|
|
assert PROVIDER_NAME_MAPPING["openai"] == "openai"
|
|
assert PROVIDER_NAME_MAPPING["anthropic"] == "anthropic"
|
|
assert PROVIDER_NAME_MAPPING["gemini"] == "google"
|
|
assert PROVIDER_NAME_MAPPING["vertexai"] == "google"
|
|
assert PROVIDER_NAME_MAPPING["groq"] == "groq"
|
|
assert PROVIDER_NAME_MAPPING["ollama"] == "ollama"
|
|
assert PROVIDER_NAME_MAPPING["openai-codex"] == "openai"
|
|
assert PROVIDER_NAME_MAPPING["claude-code"] == "anthropic"
|
|
|
|
|
|
def test_truncate_content_short():
|
|
"""Test that short content is not truncated."""
|
|
content = "This is a short message"
|
|
result = _truncate_content(content)
|
|
assert result == content
|
|
|
|
|
|
def test_truncate_content_long():
|
|
"""Test that long content is truncated."""
|
|
content = "x" * 150000 # Exceeds MAX_CONTENT_LENGTH
|
|
result = _truncate_content(content)
|
|
assert len(result) < len(content)
|
|
assert "[TRUNCATED:" in result
|
|
assert result.startswith("x" * 100)
|
|
|
|
|
|
def test_noop_span_recorder():
|
|
"""Test that NoOpLLMSpanRecorder doesn't raise errors."""
|
|
recorder = NoOpLLMSpanRecorder()
|
|
# Should not raise any errors
|
|
recorder.record_llm_call(
|
|
provider="openai",
|
|
model="gpt-4",
|
|
scope="test",
|
|
messages=[{"role": "user", "content": "test"}],
|
|
response_content="test response",
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
duration=1.0,
|
|
)
|
|
|
|
|
|
def test_llm_span_recorder_format_messages():
|
|
"""Test message formatting to GenAI convention."""
|
|
mock_tracer = MagicMock()
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful"},
|
|
{"role": "user", "content": "Hello"},
|
|
]
|
|
|
|
result = recorder._format_messages(messages)
|
|
parsed = json.loads(result)
|
|
|
|
assert len(parsed) == 2
|
|
assert parsed[0]["role"] == "system"
|
|
assert parsed[0]["content"] == "You are helpful"
|
|
assert parsed[1]["role"] == "user"
|
|
assert parsed[1]["content"] == "Hello"
|
|
|
|
|
|
def test_llm_span_recorder_format_output():
|
|
"""Test output formatting to GenAI convention."""
|
|
mock_tracer = MagicMock()
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
result = recorder._format_output("Hello world", "stop")
|
|
parsed = json.loads(result)
|
|
|
|
assert len(parsed) == 1
|
|
assert parsed[0]["role"] == "assistant"
|
|
assert parsed[0]["content"] == "Hello world"
|
|
|
|
|
|
def test_llm_span_recorder_format_output_none():
|
|
"""Test output formatting with None content."""
|
|
mock_tracer = MagicMock()
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
result = recorder._format_output(None, None)
|
|
parsed = json.loads(result)
|
|
|
|
assert parsed == []
|
|
|
|
|
|
def test_llm_span_recorder_extract_system_instructions():
|
|
"""Test system instruction extraction."""
|
|
mock_tracer = MagicMock()
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
messages = [
|
|
{"role": "system", "content": "You are helpful"},
|
|
{"role": "user", "content": "Hello"},
|
|
]
|
|
|
|
result = recorder._extract_system_instructions(messages)
|
|
assert result == "You are helpful"
|
|
|
|
|
|
def test_llm_span_recorder_extract_system_instructions_none():
|
|
"""Test system instruction extraction with no system message."""
|
|
mock_tracer = MagicMock()
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
messages = [
|
|
{"role": "user", "content": "Hello"},
|
|
]
|
|
|
|
result = recorder._extract_system_instructions(messages)
|
|
assert result is None
|
|
|
|
|
|
@patch("hindsight_api.tracing.time")
|
|
def test_llm_span_recorder_record_success(mock_time):
|
|
"""Test successful LLM call recording."""
|
|
# Mock time
|
|
mock_time.time_ns.return_value = 1000000000000 # 1 second in nanoseconds
|
|
|
|
# Create mock tracer and span
|
|
mock_span = MagicMock()
|
|
mock_tracer = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value.__enter__.return_value = mock_span
|
|
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
messages = [{"role": "user", "content": "Hello"}]
|
|
response_content = "Hi there!"
|
|
|
|
recorder.record_llm_call(
|
|
provider="openai",
|
|
model="gpt-4",
|
|
scope="test",
|
|
messages=messages,
|
|
response_content=response_content,
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
duration=1.5,
|
|
finish_reason="stop",
|
|
error=None,
|
|
)
|
|
|
|
# Verify span was created with correct name (hindsight.{scope})
|
|
mock_tracer.start_as_current_span.assert_called_once()
|
|
call_args = mock_tracer.start_as_current_span.call_args
|
|
assert call_args[0][0] == "hindsight.test"
|
|
|
|
# Verify attributes were set
|
|
assert mock_span.set_attribute.called
|
|
attribute_calls = {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list}
|
|
|
|
assert attribute_calls[GenAIAttributes.OPERATION_NAME] == "chat"
|
|
assert attribute_calls[GenAIAttributes.PROVIDER_NAME] == "openai"
|
|
assert attribute_calls[GenAIAttributes.REQUEST_MODEL] == "gpt-4"
|
|
assert attribute_calls[GenAIAttributes.RESPONSE_MODEL] == "gpt-4"
|
|
assert attribute_calls[GenAIAttributes.USAGE_INPUT_TOKENS] == 10
|
|
assert attribute_calls[GenAIAttributes.USAGE_OUTPUT_TOKENS] == 5
|
|
assert attribute_calls["hindsight.scope"] == "test"
|
|
|
|
# Verify event was added
|
|
mock_span.add_event.assert_called_once()
|
|
event_call = mock_span.add_event.call_args
|
|
assert event_call[0][0] == "gen_ai.client.inference.operation.details"
|
|
|
|
# Verify status was set to OK
|
|
mock_span.set_status.assert_called()
|
|
|
|
# Verify span was ended
|
|
mock_span.end.assert_called_once()
|
|
|
|
|
|
@patch("hindsight_api.tracing.time")
|
|
def test_llm_span_recorder_record_error(mock_time):
|
|
"""Test error LLM call recording."""
|
|
# Mock time
|
|
mock_time.time_ns.return_value = 1000000000000
|
|
|
|
# Create mock tracer and span
|
|
mock_span = MagicMock()
|
|
mock_tracer = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value.__enter__.return_value = mock_span
|
|
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
messages = [{"role": "user", "content": "Hello"}]
|
|
error = ValueError("Test error")
|
|
|
|
recorder.record_llm_call(
|
|
provider="anthropic",
|
|
model="claude-3",
|
|
scope="test",
|
|
messages=messages,
|
|
response_content=None,
|
|
input_tokens=10,
|
|
output_tokens=0,
|
|
duration=0.5,
|
|
finish_reason=None,
|
|
error=error,
|
|
)
|
|
|
|
# Verify error status was set
|
|
mock_span.set_status.assert_called()
|
|
status_call = mock_span.set_status.call_args[0][0]
|
|
assert status_call.status_code.name == "ERROR"
|
|
|
|
# Verify error type attribute was set
|
|
attribute_calls = {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list}
|
|
assert attribute_calls[GenAIAttributes.ERROR_TYPE] == "ValueError"
|
|
|
|
# Verify exception was recorded
|
|
mock_span.record_exception.assert_called_once_with(error)
|
|
|
|
|
|
@patch("hindsight_api.tracing.time")
|
|
def test_llm_span_recorder_provider_mapping(mock_time):
|
|
"""Test that provider names are mapped correctly."""
|
|
mock_time.time_ns.return_value = 1000000000000
|
|
|
|
mock_span = MagicMock()
|
|
mock_tracer = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value.__enter__.return_value = mock_span
|
|
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
|
|
# Test gemini -> google mapping
|
|
recorder.record_llm_call(
|
|
provider="gemini",
|
|
model="gemini-pro",
|
|
scope="test",
|
|
messages=[{"role": "user", "content": "test"}],
|
|
response_content="test",
|
|
input_tokens=5,
|
|
output_tokens=3,
|
|
duration=1.0,
|
|
)
|
|
|
|
attribute_calls = {call[0][0]: call[0][1] for call in mock_span.set_attribute.call_args_list}
|
|
assert attribute_calls[GenAIAttributes.PROVIDER_NAME] == "google"
|
|
|
|
|
|
# ==================== Parent Span Tests ====================
|
|
|
|
|
|
@patch("hindsight_api.tracing._tracing_enabled", False)
|
|
def test_create_operation_span_disabled():
|
|
"""Test that create_operation_span returns no-op when tracing is disabled."""
|
|
# Tracing should be disabled by default (explicitly patched for test isolation)
|
|
assert not is_tracing_enabled()
|
|
|
|
# Should return a no-op context manager
|
|
span = create_operation_span("test_operation", "test_bank_id")
|
|
|
|
# Should be usable as context manager without errors
|
|
with span:
|
|
pass
|
|
|
|
|
|
@patch("hindsight_api.tracing._tracer")
|
|
@patch("hindsight_api.tracing._tracing_enabled", True)
|
|
def test_create_operation_span_enabled(mock_tracer):
|
|
"""Test that create_operation_span creates a span when tracing is enabled."""
|
|
# Mock the tracer
|
|
mock_span = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value = mock_span
|
|
|
|
# Create operation span
|
|
span = create_operation_span("retain", "bank123")
|
|
|
|
# Verify span was created with correct name
|
|
mock_tracer.start_as_current_span.assert_called_once_with("hindsight.retain")
|
|
|
|
# Verify attributes were set
|
|
mock_span.set_attribute.assert_any_call("hindsight.operation", "retain")
|
|
mock_span.set_attribute.assert_any_call("hindsight.bank_id", "bank123")
|
|
|
|
|
|
@patch("hindsight_api.tracing._tracer")
|
|
@patch("hindsight_api.tracing._tracing_enabled", True)
|
|
def test_create_operation_span_no_bank_id(mock_tracer):
|
|
"""Test that create_operation_span works without bank_id."""
|
|
mock_span = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value = mock_span
|
|
|
|
# Create operation span without bank_id
|
|
span = create_operation_span("consolidation")
|
|
|
|
# Verify span was created
|
|
mock_tracer.start_as_current_span.assert_called_once_with("hindsight.consolidation")
|
|
|
|
# Verify only operation attribute was set (not bank_id)
|
|
assert mock_span.set_attribute.call_count == 1
|
|
mock_span.set_attribute.assert_called_once_with("hindsight.operation", "consolidation")
|
|
|
|
|
|
@patch("hindsight_api.tracing._tracer")
|
|
@patch("hindsight_api.tracing._tracing_enabled", True)
|
|
def test_create_operation_span_all_operations(mock_tracer):
|
|
"""Test that all 4 operations can create parent spans."""
|
|
mock_span = MagicMock()
|
|
mock_tracer.start_as_current_span.return_value = mock_span
|
|
|
|
operations = ["retain", "consolidation", "reflect", "mental_model_refresh"]
|
|
|
|
for operation in operations:
|
|
mock_tracer.reset_mock()
|
|
mock_span.reset_mock()
|
|
|
|
span = create_operation_span(operation, "test_bank")
|
|
|
|
# Verify span was created with correct name
|
|
mock_tracer.start_as_current_span.assert_called_once_with(f"hindsight.{operation}")
|
|
|
|
# Verify attributes
|
|
mock_span.set_attribute.assert_any_call("hindsight.operation", operation)
|
|
mock_span.set_attribute.assert_any_call("hindsight.bank_id", "test_bank")
|
|
|
|
|
|
@patch("hindsight_api.tracing.time")
|
|
@patch("hindsight_api.tracing._tracer")
|
|
@patch("hindsight_api.tracing._tracing_enabled", True)
|
|
def test_parent_child_span_hierarchy(mock_tracer, mock_time):
|
|
"""Test that child LLM spans are created under parent operation spans."""
|
|
mock_time.time_ns.return_value = 1000000000000
|
|
|
|
# Create mock parent span
|
|
mock_parent_span = MagicMock()
|
|
mock_parent_span.__enter__ = MagicMock(return_value=mock_parent_span)
|
|
mock_parent_span.__exit__ = MagicMock(return_value=False)
|
|
|
|
# Create mock child span
|
|
mock_child_span = MagicMock()
|
|
|
|
# Mock tracer to return parent span first, then child span
|
|
mock_tracer.start_as_current_span.side_effect = [
|
|
mock_parent_span, # Parent span
|
|
MagicMock(__enter__=MagicMock(return_value=mock_child_span), __exit__=MagicMock(return_value=False)), # Child
|
|
]
|
|
|
|
# Create parent operation span
|
|
with create_operation_span("retain", "bank123"):
|
|
# Simulate creating a child LLM span
|
|
recorder = LLMSpanRecorder(mock_tracer)
|
|
recorder.record_llm_call(
|
|
provider="openai",
|
|
model="gpt-4",
|
|
scope="retain_extract_facts",
|
|
messages=[{"role": "user", "content": "test"}],
|
|
response_content="response",
|
|
input_tokens=10,
|
|
output_tokens=5,
|
|
duration=1.0,
|
|
)
|
|
|
|
# Verify both parent and child spans were created
|
|
assert mock_tracer.start_as_current_span.call_count == 2
|
|
|
|
# Verify parent span was created first
|
|
first_call = mock_tracer.start_as_current_span.call_args_list[0]
|
|
assert first_call[0][0] == "hindsight.retain"
|
|
|
|
# Verify child span was created second (hindsight.{scope})
|
|
second_call = mock_tracer.start_as_current_span.call_args_list[1]
|
|
assert second_call[0][0] == "hindsight.retain_extract_facts"
|
|
|
|
|
|
@patch("hindsight_api.tracing._tracer")
|
|
@patch("hindsight_api.tracing._tracing_enabled", True)
|
|
def test_operation_span_context_manager(mock_tracer):
|
|
"""Test that operation spans work as context managers."""
|
|
mock_span = MagicMock()
|
|
mock_span.__enter__ = MagicMock(return_value=mock_span)
|
|
mock_span.__exit__ = MagicMock(return_value=False)
|
|
mock_tracer.start_as_current_span.return_value = mock_span
|
|
|
|
# Use span as context manager
|
|
with create_operation_span("reflect", "bank456"):
|
|
# Do some work
|
|
pass
|
|
|
|
# Verify span lifecycle
|
|
mock_tracer.start_as_current_span.assert_called_once()
|
|
mock_span.__enter__.assert_called_once()
|
|
mock_span.__exit__.assert_called_once()
|