hermes-agent/tests/agent/test_auxiliary_reasoning_field_rejection_retry.py
teknium1 65aaf16a4d fix(aux): only treat a standalone reasoning field name as a reasoning rejection; let parameter rungs chain
Review follow-up on #113114. `_is_reasoning_field_rejection` fired on any 400 whose text
contained "reasoning"/"think" next to a generic unsupported marker, so a route-gating 400
naming a thinking model ("The model kimi-k2-thinking is not supported when using this
account") spent a strip-retry and then re-raised from `_param_rung_accepts` — the configured
fallback chain, which main consulted for that error, was never reached. Require the token
to be a standalone wire-field name (not a model-id segment, not "... with reasoning
models"), and let `_param_rung_accepts` accept model-incompatible, reasoning-field and
structured-output rejections so a temperature-strip retry that 400s on `reasoning_effort`
reaches the reasoning rung and a gating 400 after any strip still reaches provider fallback.

Also mirrors the new retry paragraph into the zh-Hans configuration page.
2026-09-16 17:01:52 -07:00

97 lines
4.8 KiB
Python

"""Reasoning-field rejection retry in ``agent.auxiliary_client`` (#112781).
The title lane disables reasoning (``reasoning_config={"enabled": False}``); on
``provider=custom`` the profile encodes that as top-level ``reasoning_effort: "none"``
(the deliberate Ollama /v1 / vLLM / GLM thinking-off wire). A chat-only model behind an
OpenAI-compatible relay (gpt-4.1-mini on a one-api style relay) answers
``400 Unrecognized request argument supplied: reasoning_effort`` and, before the fix,
the title was lost with no retry. The recovery is reactive like the temperature and
``response_format`` rungs: strip every reasoning field and retry once. The custom
profile's encoding itself is untouched so Ollama/vLLM/GLM users keep thinking-off.
"""
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from agent.auxiliary_client import async_call_llm, call_llm
_RELAY_400 = (
"Error code: 400 - {'error': {'message': 'Unrecognized request argument supplied: "
"reasoning_effort', 'type': 'invalid_request_error', 'param': '', 'code': None}}"
)
def _custom_route_patches(client):
return (
patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("custom", "gpt-4.1-mini", "https://relay.example/v1", "sk-x", None)),
patch("agent.auxiliary_client._get_cached_client", return_value=(client, "gpt-4.1-mini")),
patch("agent.auxiliary_client._validate_llm_response", side_effect=lambda resp, _task, **_kw: resp),
patch("agent.auxiliary_client._try_payment_fallback", return_value=None),
)
def _call(async_mode, client, **kw):
common = dict(task="title_generation", messages=[{"role": "user", "content": "hi"}],
extra_body={"response_format": {"type": "json_object"}},
reasoning_config={"enabled": False}, **kw)
p1, p2, p3, p4 = _custom_route_patches(client)
with p1, p2, p3, p4:
if async_mode:
return asyncio.run(async_call_llm(**common))
return call_llm(**common)
@pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"])
def test_reasoning_effort_rejection_retries_once_without_reasoning_fields(async_mode):
"""Relay 400 naming ``reasoning_effort`` → one retry with no reasoning field; title lands."""
client = MagicMock()
client.base_url = "https://relay.example/v1"
side_effect = [RuntimeError(_RELAY_400), {"ok": True}]
client.chat.completions.create = AsyncMock(side_effect=side_effect) if async_mode else MagicMock(side_effect=side_effect)
assert _call(async_mode, client) == {"ok": True}
calls = client.chat.completions.create.call_args_list
assert len(calls) == 2
first, retry = calls[0].kwargs, calls[1].kwargs
assert first["reasoning_effort"] == "none" # the deliberate thinking-off encoding still goes out first
assert "reasoning_effort" not in retry
assert "reasoning" not in (retry.get("extra_body") or {})
assert retry["extra_body"]["response_format"] == {"type": "json_object"} # unrelated fields survive
assert retry["model"] == first["model"]
def test_unrelated_400_does_not_strip_reasoning_fields():
"""A 400 that does not name a reasoning field must not silently drop the thinking-off encoding."""
client = MagicMock()
client.base_url = "https://relay.example/v1"
client.chat.completions.create.side_effect = RuntimeError(
"HTTP 400: Invalid value: 'tool'. Supported values are: 'assistant'")
with pytest.raises(RuntimeError, match="Invalid value"):
_call(False, client)
assert client.chat.completions.create.call_count == 1
def test_model_gating_400_naming_a_thinking_model_still_reaches_the_fallback_chain():
"""A route-gating 400 whose text merely contains a reasoning token inside the model id
("kimi-k2-thinking is not supported when using this account") is not a field rejection: no
strip-retry is spent on it and the configured fallback chain is consulted exactly as on main."""
client = MagicMock()
client.base_url = "https://relay.example/v1"
client.chat.completions.create.side_effect = RuntimeError(
"Error code: 400 - The model kimi-k2-thinking is not supported when using this account")
fb_client = MagicMock()
fb_client.chat.completions.create.return_value = {"fb": True}
p1, p2, p3, _p4 = _custom_route_patches(client)
with p1, p2, p3, patch("agent.auxiliary_client._try_configured_fallback_chain",
return_value=(fb_client, "fallback-model", "fallback")) as fallback:
result = call_llm(task="title_generation", messages=[{"role": "user", "content": "hi"}],
reasoning_config={"enabled": False})
assert result == {"fb": True}
assert client.chat.completions.create.call_count == 1 # no wasted reasoning-strip retry
assert fallback.called