15ea23d5d6
* feat: introduce hindsight-api-slim and hindsight-all-slim packages Closes #552 - Move all source code from hindsight-api/ to new hindsight-api-slim/ - hindsight-api-slim has heavy ML deps (torch, sentence-transformers, transformers, einops, flashrank, mlx, mlx-lm, safetensors) and pg0-embedded as optional extras: [local-ml], [embedded-db], [all] - hindsight-api becomes a zero-code meta-package depending on hindsight-api-slim[all] for full backward compatibility - Add hindsight-all-slim meta-package: hindsight-api-slim + client + embed - hindsight-all updated to depend on hindsight-api-slim[all] - pg0.py: lazy-import pg0 with clear ImportError pointing to [embedded-db] - Dockerfile: replace sed hack with proper uv sync --extra flags - Update release.yml, test.yml, lint.sh, release.sh, CLAUDE.md and all path references throughout the repo * refactor: rename hindsight/ directory to hindsight-all/ * docs: document hindsight-api-slim and hindsight-all-slim package variants Add package variants table and extras explanation to installation.md * docs: remove emojis from installation.md, use professional tone * docs: link Docker slim variant to pip package variants section * docs: consolidate Docker image variants into single table * ci: fix working-directory paths after package restructure - Replace all hindsight-api → hindsight-api-slim in test.yml - Replace hindsight → hindsight-all in test.yml - Add --extra embedded-db to test-embed API install step * ci: add local-ml and embedded-db extras to API sync steps These extras were previously implicit in the old hindsight-api package (which bundled everything). Now that hindsight-api-slim uses optional extras, we must explicitly request local-ml and embedded-db in CI. * ci: add API install step with embedded-db to test-embed smoke test The smoke test starts hindsight-api as a daemon, which requires pg0-embedded. Add a dedicated install step for hindsight-api-slim with embedded-db extra so the daemon can start successfully. * ci: remove --no-install-project when using optional extras When --no-install-project is combined with --extra, the optional deps are not installed because extras require the project to be active. Remove --no-install-project from steps that need local-ml or embedded-db. * ci: fix ordering of uv sync steps to preserve optional extras When uv sync runs for a different workspace member, it removes optional extras installed for other members. Fix by always running extra-requiring API sync last, after other workspace member syncs. Also remove --no-install-project from embedded-db sync in test-embed, as --no-install-project prevents optional extras from being active. * ci: add local-ml extra to test-embed API install for smoke test The smoke test starts the full API server which needs sentence-transformers for local embeddings (default provider). Add local-ml extra to the install. * ci: simplify extras with --all-extras and add slim pip smoke test - Replace explicit --extra local-ml --extra embedded-db with --all-extras for cleaner, more maintainable sync steps - Add test-pip-slim job: tests hindsight-api-slim[embedded-db] without local ML models, using Cohere for embeddings/reranking (mirrors Docker slim smoke test approach) * ci: simplify slim smoke test to health check only (mirrors Docker test)
329 lines
11 KiB
Python
329 lines
11 KiB
Python
"""
|
|
Tests for LLM tool calling functionality.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from hindsight_api.engine.llm_wrapper import LLMProvider
|
|
from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult
|
|
|
|
|
|
# Sample tools for testing
|
|
SAMPLE_TOOLS = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"description": "Get weather for a location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {"type": "string", "description": "City name"},
|
|
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
|
},
|
|
"required": ["location"],
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "search",
|
|
"description": "Search for information",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {"type": "string", "description": "Search query"},
|
|
},
|
|
"required": ["query"],
|
|
},
|
|
},
|
|
},
|
|
]
|
|
|
|
|
|
class TestMockToolCalling:
|
|
"""Test tool calling with mock provider."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_call_with_tools_returns_tool_calls(self):
|
|
"""Test that mock provider can return tool calls."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
# Set mock response to return tool calls
|
|
llm.set_mock_response([
|
|
{"name": "get_weather", "arguments": {"location": "Paris", "unit": "celsius"}},
|
|
])
|
|
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "What's the weather in Paris?"}],
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
assert isinstance(result, LLMToolCallResult)
|
|
assert len(result.tool_calls) == 1
|
|
assert result.tool_calls[0].name == "get_weather"
|
|
assert result.tool_calls[0].arguments == {"location": "Paris", "unit": "celsius"}
|
|
assert result.finish_reason == "tool_calls"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_call_with_tools_returns_content(self):
|
|
"""Test that mock provider can return plain content."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
# Default mock response is plain content
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
assert isinstance(result, LLMToolCallResult)
|
|
assert result.content == "mock response"
|
|
assert len(result.tool_calls) == 0
|
|
assert result.finish_reason == "stop"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_call_with_tools_records_calls(self):
|
|
"""Test that mock calls are recorded."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
llm.clear_mock_calls()
|
|
|
|
await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "Test message"}],
|
|
tools=SAMPLE_TOOLS,
|
|
scope="test_scope",
|
|
)
|
|
|
|
calls = llm.get_mock_calls()
|
|
assert len(calls) == 1
|
|
assert calls[0]["scope"] == "test_scope"
|
|
assert "get_weather" in calls[0]["tools"]
|
|
assert "search" in calls[0]["tools"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_call_with_tools_multiple_tool_calls(self):
|
|
"""Test handling multiple tool calls in one response."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
llm.set_mock_response([
|
|
{"name": "get_weather", "arguments": {"location": "Paris"}},
|
|
{"name": "search", "arguments": {"query": "weather forecast"}},
|
|
])
|
|
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "Weather in Paris and search for forecasts"}],
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
assert len(result.tool_calls) == 2
|
|
assert result.tool_calls[0].name == "get_weather"
|
|
assert result.tool_calls[1].name == "search"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_call_with_tools_accepts_llm_tool_call_result(self):
|
|
"""Test that mock can accept LLMToolCallResult directly."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
expected_result = LLMToolCallResult(
|
|
content="Here's the info",
|
|
tool_calls=[LLMToolCall(id="call_123", name="search", arguments={"query": "test"})],
|
|
finish_reason="tool_calls",
|
|
)
|
|
llm.set_mock_response(expected_result)
|
|
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "Search for test"}],
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
assert result == expected_result
|
|
|
|
|
|
class TestToolCallConversation:
|
|
"""Test tool call conversation flow."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tool_result_message_format(self):
|
|
"""Test that tool result messages can be passed in subsequent calls."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
# First call returns tool call
|
|
llm.set_mock_response([{"name": "get_weather", "arguments": {"location": "Paris"}}])
|
|
|
|
result1 = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "What's the weather?"}],
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
# Build conversation with tool result
|
|
messages = [
|
|
{"role": "user", "content": "What's the weather?"},
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": result1.tool_calls[0].id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": result1.tool_calls[0].name,
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": result1.tool_calls[0].id,
|
|
"content": '{"temperature": 20, "conditions": "sunny"}',
|
|
},
|
|
]
|
|
|
|
# Second call should work with tool result in history
|
|
llm.set_mock_response(None) # Reset to default
|
|
result2 = await llm.call_with_tools(
|
|
messages=messages,
|
|
tools=SAMPLE_TOOLS,
|
|
)
|
|
|
|
assert result2.content == "mock response"
|
|
|
|
|
|
class TestToolSchemas:
|
|
"""Test tool schema handling."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_tools_list(self):
|
|
"""Test calling with empty tools list."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "Hello"}],
|
|
tools=[],
|
|
)
|
|
|
|
assert result.content == "mock response"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tool_with_no_required_params(self):
|
|
"""Test tool with no required parameters."""
|
|
llm = LLMProvider(provider="mock", api_key="", base_url="", model="mock")
|
|
|
|
tools = [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "list_items",
|
|
"description": "List all items",
|
|
"parameters": {"type": "object", "properties": {}, "required": []},
|
|
},
|
|
}
|
|
]
|
|
|
|
llm.set_mock_response([{"name": "list_items", "arguments": {}}])
|
|
|
|
result = await llm.call_with_tools(
|
|
messages=[{"role": "user", "content": "List items"}],
|
|
tools=tools,
|
|
)
|
|
|
|
assert len(result.tool_calls) == 1
|
|
assert result.tool_calls[0].name == "list_items"
|
|
assert result.tool_calls[0].arguments == {}
|
|
|
|
|
|
class TestReflectToolSchemas:
|
|
"""Test reflect agent tool schemas."""
|
|
|
|
def test_get_reflect_tools_default(self):
|
|
"""Test getting default reflect tools."""
|
|
from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
|
|
|
|
tools = get_reflect_tools()
|
|
|
|
tool_names = [t["function"]["name"] for t in tools]
|
|
assert "search_mental_models" in tool_names
|
|
assert "search_observations" in tool_names
|
|
assert "recall" in tool_names
|
|
assert "expand" in tool_names
|
|
assert "done" in tool_names
|
|
|
|
def test_get_reflect_tools_with_directives(self):
|
|
"""Test getting reflect tools with directive rules."""
|
|
from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
|
|
|
|
tools = get_reflect_tools(directive_rules=["Always respond in French"])
|
|
|
|
tool_names = [t["function"]["name"] for t in tools]
|
|
assert "recall" in tool_names
|
|
assert "done" in tool_names
|
|
|
|
# Done tool should have directive_compliance field when directives are present
|
|
done_tool = next(t for t in tools if t["function"]["name"] == "done")
|
|
params = done_tool["function"]["parameters"]["properties"]
|
|
assert "directive_compliance" in params
|
|
|
|
def test_get_reflect_tools_answer_mode(self):
|
|
"""Test getting reflect tools with answer output mode."""
|
|
from hindsight_api.engine.reflect.tools_schema import get_reflect_tools
|
|
|
|
tools = get_reflect_tools()
|
|
|
|
done_tool = next(t for t in tools if t["function"]["name"] == "done")
|
|
params = done_tool["function"]["parameters"]["properties"]
|
|
|
|
assert "answer" in params
|
|
assert "memory_ids" in params
|
|
assert "observation_ids" in params
|
|
assert "mental_model_ids" in params
|
|
|
|
|
|
class TestLLMToolCallResult:
|
|
"""Test LLMToolCallResult model."""
|
|
|
|
def test_tool_call_result_defaults(self):
|
|
"""Test default values for LLMToolCallResult."""
|
|
result = LLMToolCallResult()
|
|
|
|
assert result.content is None
|
|
assert result.tool_calls == []
|
|
assert result.finish_reason is None
|
|
|
|
def test_tool_call_result_with_content(self):
|
|
"""Test LLMToolCallResult with content."""
|
|
result = LLMToolCallResult(content="Hello", finish_reason="stop")
|
|
|
|
assert result.content == "Hello"
|
|
assert result.tool_calls == []
|
|
assert result.finish_reason == "stop"
|
|
|
|
def test_tool_call_result_with_tool_calls(self):
|
|
"""Test LLMToolCallResult with tool calls."""
|
|
result = LLMToolCallResult(
|
|
tool_calls=[
|
|
LLMToolCall(id="call_1", name="test_tool", arguments={"arg": "value"}),
|
|
],
|
|
finish_reason="tool_calls",
|
|
)
|
|
|
|
assert result.content is None
|
|
assert len(result.tool_calls) == 1
|
|
assert result.tool_calls[0].name == "test_tool"
|
|
assert result.finish_reason == "tool_calls"
|
|
|
|
|
|
class TestLLMToolCall:
|
|
"""Test LLMToolCall model."""
|
|
|
|
def test_tool_call_basic(self):
|
|
"""Test basic LLMToolCall creation."""
|
|
call = LLMToolCall(id="call_123", name="get_weather", arguments={"location": "Paris"})
|
|
|
|
assert call.id == "call_123"
|
|
assert call.name == "get_weather"
|
|
assert call.arguments == {"location": "Paris"}
|
|
|
|
def test_tool_call_empty_arguments(self):
|
|
"""Test LLMToolCall with empty arguments."""
|
|
call = LLMToolCall(id="call_456", name="list_items", arguments={})
|
|
|
|
assert call.arguments == {}
|