mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
Review follow-up on #113114. `_is_reasoning_field_rejection` fired on any 400 whose text contained "reasoning"/"think" next to a generic unsupported marker, so a route-gating 400 naming a thinking model ("The model kimi-k2-thinking is not supported when using this account") spent a strip-retry and then re-raised from `_param_rung_accepts` — the configured fallback chain, which main consulted for that error, was never reached. Require the token to be a standalone wire-field name (not a model-id segment, not "... with reasoning models"), and let `_param_rung_accepts` accept model-incompatible, reasoning-field and structured-output rejections so a temperature-strip retry that 400s on `reasoning_effort` reaches the reasoning rung and a gating 400 after any strip still reaches provider fallback. Also mirrors the new retry paragraph into the zh-Hans configuration page.
97 lines
4.8 KiB
Python
97 lines
4.8 KiB
Python
"""Reasoning-field rejection retry in ``agent.auxiliary_client`` (#112781).
|
|
|
|
The title lane disables reasoning (``reasoning_config={"enabled": False}``); on
|
|
``provider=custom`` the profile encodes that as top-level ``reasoning_effort: "none"``
|
|
(the deliberate Ollama /v1 / vLLM / GLM thinking-off wire). A chat-only model behind an
|
|
OpenAI-compatible relay (gpt-4.1-mini on a one-api style relay) answers
|
|
``400 Unrecognized request argument supplied: reasoning_effort`` and, before the fix,
|
|
the title was lost with no retry. The recovery is reactive like the temperature and
|
|
``response_format`` rungs: strip every reasoning field and retry once. The custom
|
|
profile's encoding itself is untouched so Ollama/vLLM/GLM users keep thinking-off.
|
|
"""
|
|
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.auxiliary_client import async_call_llm, call_llm
|
|
|
|
_RELAY_400 = (
|
|
"Error code: 400 - {'error': {'message': 'Unrecognized request argument supplied: "
|
|
"reasoning_effort', 'type': 'invalid_request_error', 'param': '', 'code': None}}"
|
|
)
|
|
|
|
|
|
def _custom_route_patches(client):
|
|
return (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("custom", "gpt-4.1-mini", "https://relay.example/v1", "sk-x", None)),
|
|
patch("agent.auxiliary_client._get_cached_client", return_value=(client, "gpt-4.1-mini")),
|
|
patch("agent.auxiliary_client._validate_llm_response", side_effect=lambda resp, _task, **_kw: resp),
|
|
patch("agent.auxiliary_client._try_payment_fallback", return_value=None),
|
|
)
|
|
|
|
|
|
def _call(async_mode, client, **kw):
|
|
common = dict(task="title_generation", messages=[{"role": "user", "content": "hi"}],
|
|
extra_body={"response_format": {"type": "json_object"}},
|
|
reasoning_config={"enabled": False}, **kw)
|
|
p1, p2, p3, p4 = _custom_route_patches(client)
|
|
with p1, p2, p3, p4:
|
|
if async_mode:
|
|
return asyncio.run(async_call_llm(**common))
|
|
return call_llm(**common)
|
|
|
|
|
|
@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"])
|
|
def test_reasoning_effort_rejection_retries_once_without_reasoning_fields(async_mode):
|
|
"""Relay 400 naming ``reasoning_effort`` → one retry with no reasoning field; title lands."""
|
|
client = MagicMock()
|
|
client.base_url = "https://relay.example/v1"
|
|
side_effect = [RuntimeError(_RELAY_400), {"ok": True}]
|
|
client.chat.completions.create = AsyncMock(side_effect=side_effect) if async_mode else MagicMock(side_effect=side_effect)
|
|
|
|
assert _call(async_mode, client) == {"ok": True}
|
|
|
|
calls = client.chat.completions.create.call_args_list
|
|
assert len(calls) == 2
|
|
first, retry = calls[0].kwargs, calls[1].kwargs
|
|
assert first["reasoning_effort"] == "none" # the deliberate thinking-off encoding still goes out first
|
|
assert "reasoning_effort" not in retry
|
|
assert "reasoning" not in (retry.get("extra_body") or {})
|
|
assert retry["extra_body"]["response_format"] == {"type": "json_object"} # unrelated fields survive
|
|
assert retry["model"] == first["model"]
|
|
|
|
|
|
def test_unrelated_400_does_not_strip_reasoning_fields():
|
|
"""A 400 that does not name a reasoning field must not silently drop the thinking-off encoding."""
|
|
client = MagicMock()
|
|
client.base_url = "https://relay.example/v1"
|
|
client.chat.completions.create.side_effect = RuntimeError(
|
|
"HTTP 400: Invalid value: 'tool'. Supported values are: 'assistant'")
|
|
|
|
with pytest.raises(RuntimeError, match="Invalid value"):
|
|
_call(False, client)
|
|
assert client.chat.completions.create.call_count == 1
|
|
|
|
|
|
def test_model_gating_400_naming_a_thinking_model_still_reaches_the_fallback_chain():
|
|
"""A route-gating 400 whose text merely contains a reasoning token inside the model id
|
|
("kimi-k2-thinking is not supported when using this account") is not a field rejection: no
|
|
strip-retry is spent on it and the configured fallback chain is consulted exactly as on main."""
|
|
client = MagicMock()
|
|
client.base_url = "https://relay.example/v1"
|
|
client.chat.completions.create.side_effect = RuntimeError(
|
|
"Error code: 400 - The model kimi-k2-thinking is not supported when using this account")
|
|
fb_client = MagicMock()
|
|
fb_client.chat.completions.create.return_value = {"fb": True}
|
|
p1, p2, p3, _p4 = _custom_route_patches(client)
|
|
with p1, p2, p3, patch("agent.auxiliary_client._try_configured_fallback_chain",
|
|
return_value=(fb_client, "fallback-model", "fallback")) as fallback:
|
|
result = call_llm(task="title_generation", messages=[{"role": "user", "content": "hi"}],
|
|
reasoning_config={"enabled": False})
|
|
|
|
assert result == {"fb": True}
|
|
assert client.chat.completions.create.call_count == 1 # no wasted reasoning-strip retry
|
|
assert fallback.called
|