diff --git a/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py b/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py index 6bc08ed2..01f70269 100644 --- a/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py +++ b/hindsight-api/hindsight_api/engine/providers/openai_compatible_llm.py @@ -522,6 +522,19 @@ class OpenAICompatibleLLM(LLMInterface): """ start_time = time.time() + # Normalize named tool_choice dicts to "required" + filter tools. + # Some providers (e.g. LM Studio, Ollama) reject the OpenAI named format + # {"type": "function", "function": {"name": "..."}}. The semantics are + # identical to tool_choice="required" with the tools list restricted to + # just the requested tool, so we apply that transformation universally. + if isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + forced_name = tool_choice.get("function", {}).get("name") + if forced_name: + filtered = [t for t in tools if t.get("function", {}).get("name") == forced_name] + if filtered: + tools = filtered + tool_choice = "required" + # Build call parameters call_params: dict[str, Any] = { "model": self.model, diff --git a/hindsight-api/tests/test_lmstudio_tool_choice.py b/hindsight-api/tests/test_lmstudio_tool_choice.py new file mode 100644 index 00000000..221fc93d --- /dev/null +++ b/hindsight-api/tests/test_lmstudio_tool_choice.py @@ -0,0 +1,321 @@ +""" +Reproduce issue #520: Reflect fails with LM Studio due to unsupported tool_choice format. + +The reflect agent forces tool selection via named tool_choice dicts on the first few iterations: + {"type": "function", "function": {"name": "search_mental_models"}} + +LM Studio (and Ollama) reject this format with HTTP 400: + "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'." + +The fix should convert named tool_choice to "required" and filter the tools list +to only the requested tool for providers that don't support named tool_choice. +""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from openai import APIStatusError + +from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM + +# Reflect agent tools (subset matching what agent.py uses) +REFLECT_TOOLS = [ + { + "type": "function", + "function": { + "name": "search_mental_models", + "description": "Search consolidated mental models", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_observations", + "description": "Search raw observations", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "recall", + "description": "Recall semantic memories", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "done", + "description": "Finish and return the answer", + "parameters": { + "type": "object", + "properties": {"answer": {"type": "string"}}, + "required": ["answer"], + }, + }, + }, +] + + +def _make_lmstudio_llm() -> OpenAICompatibleLLM: + return OpenAICompatibleLLM( + provider="lmstudio", + api_key="local", + base_url="http://localhost:1234/v1", + model="openai/gpt-oss-20b", + ) + + +def _lmstudio_400_error(msg: str = "Tool choice of type 'function' is not supported. Use 'auto', 'none', or 'required'.") -> APIStatusError: + """Simulate the HTTP 400 LM Studio returns for unsupported tool_choice format.""" + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.headers = {} + return APIStatusError( + message=msg, + response=mock_response, + body={"error": {"message": msg, "type": "invalid_request_error"}}, + ) + + +def _make_tool_call_response(tool_name: str, arguments: dict) -> MagicMock: + """Build a mock successful tool call response from the LLM API.""" + mock_tc = MagicMock() + mock_tc.id = "call_abc123" + mock_tc.function.name = tool_name + mock_tc.function.arguments = json.dumps(arguments) + + mock_response = MagicMock() + mock_response.usage.prompt_tokens = 120 + mock_response.usage.completion_tokens = 40 + mock_response.usage.total_tokens = 160 + mock_response.choices[0].finish_reason = "tool_calls" + mock_response.choices[0].message.content = None + mock_response.choices[0].message.tool_calls = [mock_tc] + return mock_response + + +class TestLMStudioNamedToolChoiceBug: + """ + Reproduces issue #520. + + The reflect agent (agent.py lines 546-555) sets tool_choice to a named dict + on the first iterations to force sequential retrieval: + + iteration=0, has_mental_models=True → {"type": "function", "function": {"name": "search_mental_models"}} + iteration=0, has_mental_models=False → {"type": "function", "function": {"name": "search_observations"}} + iteration=1, has_mental_models=True → {"type": "function", "function": {"name": "search_observations"}} + iteration=1 or (2 with models) → {"type": "function", "function": {"name": "recall"}} + + LM Studio rejects these dict formats with HTTP 400. + """ + + @pytest.mark.asyncio + async def test_lmstudio_named_tool_choice_no_longer_causes_400(self): + """ + Regression test for issue #520: named tool_choice dict is converted to + "required" + filtered tools before the API call, so LM Studio never + sees the unsupported format and the 400 error no longer occurs. + """ + llm = _make_lmstudio_llm() + named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}} + success_response = _make_tool_call_response("search_mental_models", {"query": "user name"}) + + with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.return_value = success_response + + # Should succeed — no 400 because the dict is converted before sending + result = await llm.call_with_tools( + messages=[{"role": "user", "content": "What is the user's name?"}], + tools=REFLECT_TOOLS, + tool_choice=named_tool_choice, + max_retries=0, + ) + + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].name == "search_mental_models" + + sent_kwargs = mock_create.call_args.kwargs + assert sent_kwargs["tool_choice"] == "required" + assert len(sent_kwargs["tools"]) == 1 + assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "forced_tool_name", + ["search_mental_models", "search_observations", "recall"], + ) + async def test_all_reflect_forced_tools_fail_on_lmstudio(self, forced_tool_name: str): + """ + Each named tool_choice the reflect agent uses on iterations 0-2 triggers + the same 400 error on LM Studio. + """ + llm = _make_lmstudio_llm() + named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}} + + with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.side_effect = _lmstudio_400_error() + + with pytest.raises(APIStatusError) as exc_info: + await llm.call_with_tools( + messages=[{"role": "user", "content": "Test query"}], + tools=REFLECT_TOOLS, + tool_choice=named_tool_choice, + max_retries=0, + ) + + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_lmstudio_string_tool_choice_works_fine(self): + """ + String tool_choice values ("auto", "none", "required") ARE supported by LM Studio. + Only the dict format {"type": "function", "function": {"name": "..."}} fails. + This test confirms the control case works. + """ + llm = _make_lmstudio_llm() + success_response = _make_tool_call_response("search_mental_models", {"query": "user name"}) + + with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.return_value = success_response + + result = await llm.call_with_tools( + messages=[{"role": "user", "content": "What is the user's name?"}], + tools=REFLECT_TOOLS, + tool_choice="required", # string form — LM Studio accepts this + max_retries=0, + ) + + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].name == "search_mental_models" + + # Confirm "required" was sent, not a dict + sent_kwargs = mock_create.call_args.kwargs + assert sent_kwargs["tool_choice"] == "required" + + +class TestExpectedFixBehavior: + """ + Tests that document the EXPECTED behavior after the fix is applied. + + For lmstudio (and ollama) providers, when tool_choice is a named dict: + {"type": "function", "function": {"name": "search_mental_models"}} + + The fix should: + 1. Convert tool_choice to "required" + 2. Filter tools to only the requested tool + + These tests currently FAIL (because the fix is not yet implemented). + After the fix is applied, they should PASS. + """ + + @pytest.mark.asyncio + async def test_fix_converts_named_tool_choice_to_required(self): + """ + After fix: named tool_choice dict is converted to "required" for lmstudio. + The API receives tool_choice="required" instead of the unsupported dict. + """ + llm = _make_lmstudio_llm() + named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}} + success_response = _make_tool_call_response("search_mental_models", {"query": "user name"}) + + with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.return_value = success_response + + result = await llm.call_with_tools( + messages=[{"role": "user", "content": "What is the user's name?"}], + tools=REFLECT_TOOLS, + tool_choice=named_tool_choice, + max_retries=0, + ) + + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].name == "search_mental_models" + + sent_kwargs = mock_create.call_args.kwargs + # Fix: dict was converted to "required" + assert sent_kwargs["tool_choice"] == "required", ( + f"Expected tool_choice='required', got {sent_kwargs['tool_choice']!r}" + ) + # Fix: tools filtered to just the requested one + assert len(sent_kwargs["tools"]) == 1 + assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "forced_tool_name", + ["search_mental_models", "search_observations", "recall"], + ) + async def test_fix_filters_tools_to_requested_tool(self, forced_tool_name: str): + """ + After fix: tools list is filtered to only the forced tool so the model + can only call that one tool (equivalent to the named tool_choice behavior). + """ + llm = _make_lmstudio_llm() + named_tool_choice = {"type": "function", "function": {"name": forced_tool_name}} + success_response = _make_tool_call_response(forced_tool_name, {"query": "test"}) + + with patch.object(llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.return_value = success_response + + await llm.call_with_tools( + messages=[{"role": "user", "content": "Test query"}], + tools=REFLECT_TOOLS, + tool_choice=named_tool_choice, + max_retries=0, + ) + + sent_kwargs = mock_create.call_args.kwargs + assert sent_kwargs["tool_choice"] == "required" + assert len(sent_kwargs["tools"]) == 1 + assert sent_kwargs["tools"][0]["function"]["name"] == forced_tool_name + + @pytest.mark.asyncio + async def test_fix_also_applies_to_openai_provider(self): + """ + The fix is generalized: all providers convert named tool_choice to + "required" + filtered tools. OpenAI natively supports the dict format + too, so the behaviour is semantically identical either way. + """ + from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM + + openai_llm = OpenAICompatibleLLM( + provider="openai", + api_key="sk-test", + base_url="", + model="gpt-4o-mini", + ) + + named_tool_choice = {"type": "function", "function": {"name": "search_mental_models"}} + success_response = _make_tool_call_response("search_mental_models", {"query": "test"}) + + with patch.object(openai_llm._client.chat.completions, "create", new_callable=AsyncMock) as mock_create: + mock_create.return_value = success_response + + await openai_llm.call_with_tools( + messages=[{"role": "user", "content": "Test"}], + tools=REFLECT_TOOLS, + tool_choice=named_tool_choice, + max_retries=0, + ) + + sent_kwargs = mock_create.call_args.kwargs + # Generalized fix applies to OpenAI too + assert sent_kwargs["tool_choice"] == "required" + assert len(sent_kwargs["tools"]) == 1 + assert sent_kwargs["tools"][0]["function"]["name"] == "search_mental_models"