hermes-agent/tests/agent/test_auxiliary_reasoning_floor.py
teknium1 2768f31628 fix: step reasoning up to the floor when a route refuses to disable it
Endpoints that understand the reasoning field but refuse the OFF ("Reasoning is
mandatory for this endpoint and cannot be disabled" -- the Nous Portal on
gpt-6-astra) got two different wrong answers:

* auxiliary lanes (title generation sends reasoning_config={"enabled": False})
  400'd outright -- the #112781 strip rung only matched "unsupported/unknown
  field" wordings, so every session on such a route stayed untitled;
* the main loop dropped the disable and let the route pick its default effort,
  the opposite of what a thinking-off user asked for.

Both now step the effort up to the lowest level every reasoning wire accepts
(low) instead. The aux ladder gets a rung ordered before the strip
(agent/auxiliary_reasoning_floor.py: lifts reasoning_effort / extra_body.reasoning
/ _reasoning_config, memoises the (route, model) so the next thinking-off aux call
starts at the floor without the guaranteed 400); the main loop's reasoning_mandatory
recovery records whether the route said mandatory (floor) or unknown-field (drop,
unchanged). error_classifier.is_reasoning_required_rejection separates the two.

Live on the portal (proxy wire capture): title none->400->low->200 titled;
main loop none->400->low->200; second aux call on the route sends low up front.
2026-09-19 02:37:36 -07:00

76 lines
3.6 KiB
Python

"""Reasoning-required rejection → step the aux effort up to the floor and remember the route.
The title lane disables reasoning (``reasoning_config={"enabled": False}``), which the custom profile
encodes as top-level ``reasoning_effort: "none"``. Endpoints that understand the field but refuse the
disable (Nous Portal on gpt-6-astra: 400 "Reasoning is mandatory for this endpoint and cannot be
disabled") used to fail the title outright — the strip rung (#112781) never matched this wording.
The recovery is a step UP (``low``), memoised per (route, model) so the next thinking-off aux call
on that route starts at the floor without the guaranteed 400.
"""
from unittest.mock import MagicMock, patch
import pytest
from agent import auxiliary_reasoning_floor
from agent.auxiliary_client import call_llm
_PORTAL_400 = (
"Error code: 400 - {'status': 400, 'message': 'This request is not valid. Check the model name and "
"other parameters. Additional info: Reasoning is mandatory for this endpoint and cannot be disabled.'}"
)
@pytest.fixture(autouse=True)
def _fresh_memo():
auxiliary_reasoning_floor._FLOORED_ROUTES.clear()
yield
auxiliary_reasoning_floor._FLOORED_ROUTES.clear()
def _call(client):
with (
patch("agent.auxiliary_client._resolve_task_provider_model",
return_value=("custom", "openai/gpt-6-astra", "http://127.0.0.1:8765/v1", "sk-x", None)),
patch("agent.auxiliary_client._get_cached_client", return_value=(client, "openai/gpt-6-astra")),
patch("agent.auxiliary_client._validate_llm_response", side_effect=lambda resp, _task, **_kw: resp),
patch("agent.auxiliary_client._try_payment_fallback", return_value=None),
):
return call_llm(
task="title_generation", messages=[{"role": "user", "content": "hi"}],
extra_body={"response_format": {"type": "json_object"}}, reasoning_config={"enabled": False},
)
def test_reasoning_required_400_steps_effort_up_to_the_floor_and_remembers_the_route():
"""First call: ``none`` → 400 → retry at the floor with everything else intact. Second call on the
same route+model: the floor goes out up front, no 400 round-trip."""
client = MagicMock()
client.base_url = "http://127.0.0.1:8765/v1"
client.chat.completions.create.side_effect = [RuntimeError(_PORTAL_400), {"ok": True}, {"ok": True}]
assert _call(client) == {"ok": True}
first, retry = (c.kwargs for c in client.chat.completions.create.call_args_list[:2])
assert first["reasoning_effort"] == "none"
assert retry["reasoning_effort"] == auxiliary_reasoning_floor.REASONING_FLOOR_EFFORT
assert retry["extra_body"]["response_format"] == {"type": "json_object"}
assert retry["model"] == first["model"]
assert _call(client) == {"ok": True}
assert client.chat.completions.create.call_count == 3
upfront = client.chat.completions.create.call_args_list[2].kwargs
assert upfront["reasoning_effort"] == auxiliary_reasoning_floor.REASONING_FLOOR_EFFORT
def test_field_rejection_still_strips_instead_of_stepping_up():
"""A relay that does not know the field at all keeps the #112781 behaviour: the field is dropped,
nothing is remembered as a floor."""
client = MagicMock()
client.base_url = "https://relay.example/v1"
client.chat.completions.create.side_effect = [
RuntimeError("Error code: 400 - Unrecognized request argument supplied: reasoning_effort"), {"ok": True},
]
assert _call(client) == {"ok": True}
retry = client.chat.completions.create.call_args_list[1].kwargs
assert "reasoning_effort" not in retry
assert not auxiliary_reasoning_floor._FLOORED_ROUTES