hermes-agent/tests/agent/test_terminal_approval_batch.py
Hermes Agent c42c90552e fix(agent): re-gate later terminal approvals after an earlier batch failure (#113158)
Desktop terminal batching pre-collects the approval for every command in
a run before any of them executes, so the user consents to a batch in
which all commands are expected to run. When an earlier command then
fails (or is denied/blocked), that informed consent no longer describes
the world the later commands will run in — yet the executor still
consumed the pre-made decision as though nothing had happened.

- _TerminalBatch.failure_seen: set by the sequential publisher after a
  slot's failed (or blocked) result is committed — i.e. after the failure
  the model actually sees, never from a wedged worker's late result.
- consume_prepared_guard drops the prepared decision for any later slot
  once a failure is published and returns None, so the guard runs its
  live flow again: tirith scan, allowlist, and a fresh human approval
  request when the command still warrants one. Nothing is auto-denied
  and an explicit denial of the failing command remains authoritative.
- The flag is sticky for the batch: a later success must not un-stale an
  approval collected before an even earlier failure.

Success-path batching is unchanged: with no failure, prepared decisions
are consumed exactly as before (byte-for-byte the same flow).
2026-09-26 21:00:46 -05:00

325 lines
15 KiB
Python

"""Desktop batches publish consent together, never shell effects together."""
import json
import queue
import threading
import time
from contextlib import ExitStack
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from run_agent import AIAgent
from gateway.session_context import clear_session_vars
from tools import approval
from tools.thread_context import propagate_context_to_thread
def _call(call_id, command):
return SimpleNamespace(id=call_id, type="function", function=SimpleNamespace(
name="terminal", arguments=json.dumps({"command": command})))
def _agent():
from tools.terminal_tool import TERMINAL_SCHEMA
from tools.file_tools import READ_FILE_SCHEMA
with (
patch("model_tools.get_tool_definitions", return_value=[
{"type": "function", "function": schema}
for schema in (TERMINAL_SCHEMA, READ_FILE_SCHEMA)
]),
patch("model_tools.check_toolset_requirements", return_value={}),
patch("agent.process_bootstrap.OpenAI"),
patch("agent.model_metadata.fetch_model_metadata", return_value={}),
):
return AIAgent(api_key="test-key", base_url="https://openrouter.ai/api/v1",
quiet_mode=True, skip_context_files=True, skip_memory=True, platform="desktop")
@pytest.mark.parametrize("read_count", [0, 2])
@pytest.mark.parametrize("threaded_middleware", [False, True])
def test_desktop_publishes_final_commands_before_wait_and_runs_in_order(tmp_path, monkeypatch, read_count, threaded_middleware):
from tools.terminal_scope import reset_terminal_scope, set_terminal_scope
from tools.terminal_tool_lifecycle import cleanup_vm
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
monkeypatch.setenv("TERMINAL_ENV", "local")
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
monkeypatch.setattr("tools.approval_context._get_approval_mode", lambda: "manual")
monkeypatch.setattr("tools.approval._tirith_scan", lambda command: {"action": "allow"})
monkeypatch.setattr("agent.title_generator.maybe_auto_title", lambda *a, **kw: None)
nested = tmp_path / "nested"
nested.mkdir()
key = "desktop-terminal-batch"
agent = _agent()
published = queue.Queue()
approval.register_gateway_notify(key, published.put)
from tui_gateway import server
monkeypatch.setattr(server, "_sessions", {key: {
"session_key": key, "source": "desktop", "agent": agent, "cwd": str(tmp_path),
}})
tokens = server._set_session_context(key)
calls = [_call("first", "original-first"), _call("second", "original-second")]
if read_count:
source = tmp_path / "input.txt"
source.write_text("input")
calls[:0] = [SimpleNamespace(id=f"read-{i}", type="function", function=SimpleNamespace(
name="read_file", arguments=json.dumps({"path": str(source)}))) for i in range(read_count)]
commands = {"original-first": f"rm -rf absent-first; cd '{nested}'", "original-second": "rm -rf absent-second; pwd"}
policy = []
executed = []
flushed = []
messages = []
errors = []
replay_errors = []
if threaded_middleware:
def middleware(name, args, execute, **kwargs):
if name != "terminal":
return execute(args)
results = []
def dispatch():
results.append(execute(args))
try:
execute(args)
except RuntimeError as exc:
replay_errors.append(str(exc))
# Middleware may move its continuation onto a fresh thread.
thread = threading.Thread(target=dispatch, daemon=True)
thread.start()
thread.join(20)
return results[0]
monkeypatch.setattr("hermes_cli.middleware.run_tool_execution_middleware", middleware)
def pre_hook(name, args, **kwargs):
if name != "terminal":
return None, args
policy.append(args["command"])
return None, {**args, "command": commands[args["command"]]}
def started(call_id, name, args):
if name != "terminal":
return
executed.append((call_id, args["command"], list(flushed)))
agent.tool_start_callback = started
agent._flush_messages_to_session_db = lambda rows, **kw: flushed.append([r["tool_call_id"] for r in rows]) or True
def run():
try:
agent._execute_tool_calls(SimpleNamespace(tool_calls=calls), messages, key)
except BaseException as exc:
errors.append(exc)
with ExitStack() as scope, patch(
"hermes_cli.plugins._dispatch_pre_tool_call_hooks", side_effect=pre_hook
):
scope.callback(reset_terminal_scope, set_terminal_scope({"TERMINAL_ENV": "local", "TERMINAL_CWD": str(tmp_path)}))
worker = threading.Thread(target=propagate_context_to_thread(run), daemon=True)
worker.start()
try:
first = published.get(timeout=10)
second = published.get(timeout=5)
assert [first["command"], second["command"]] == list(commands.values())
assert policy == list(commands)
assert executed == []
assert approval.ack_gateway_approval(key, second["request_id"])
assert executed == []
assert approval.resolve_gateway_approval(key, "once", request_id=second["request_id"]) == 1
assert executed == []
assert approval.resolve_gateway_approval(key, "once", request_id=first["request_id"]) == 1
worker.join(timeout=15)
assert not worker.is_alive()
assert errors == []
assert [item[:2] for item in executed] == [("first", commands["original-first"]), ("second", commands["original-second"])]
assert len(replay_errors) == (2 if threaded_middleware else 0)
assert executed[1][2][-1] == [c.id for c in calls[:-1]]
assert "first" not in {call_id for flush in executed[0][2] for call_id in flush}
assert [m["tool_call_id"] for m in messages] == [c.id for c in calls]
assert str(nested) in json.loads(messages[-1]["content"])["output"]
assert approval.list_gateway_approvals(key) == []
finally:
agent.interrupt("test cleanup")
approval.unregister_gateway_notify(key)
worker.join(timeout=5)
cleanup_vm(key)
clear_session_vars(tokens)
def test_cancelled_preparation_drains_requests_without_reusing_once(tmp_path, monkeypatch):
from tools.terminal_scope import reset_terminal_scope, set_terminal_scope
from tools.terminal_tool_lifecycle import cleanup_vm
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
monkeypatch.setattr("tools.approval_context._get_approval_mode", lambda: "manual")
monkeypatch.setattr("tools.approval._tirith_scan", lambda command: {"action": "allow"})
monkeypatch.setattr("agent.title_generator.maybe_auto_title", lambda *a, **kw: None)
key = "cancelled-terminal-batch"
command = "rm -rf absent; printf executed > effect.txt"
published = queue.Queue()
release_notify = threading.Event()
messages, errors = [], []
agent = _agent()
agent._flush_messages_to_session_db = lambda *a, **kw: True
calls = [_call(call_id, command) for call_id in ("first", "second", "third")]
notices = []
def notify(data):
notices.append(data)
published.put(data)
if len(notices) == 3:
release_notify.wait(10)
def run(target, batch_calls, rows):
try:
target._execute_tool_calls(SimpleNamespace(tool_calls=batch_calls), rows, key)
except BaseException as exc:
errors.append(exc)
approval.register_gateway_notify(key, notify)
from tui_gateway import server
monkeypatch.setattr(server, "_sessions", {key: {
"session_key": key, "source": "desktop", "agent": agent, "cwd": str(tmp_path),
}})
tokens = server._set_session_context(key)
with ExitStack() as scope:
scope.callback(reset_terminal_scope, set_terminal_scope({"TERMINAL_ENV": "local", "TERMINAL_CWD": str(tmp_path)}))
worker = threading.Thread(target=propagate_context_to_thread(lambda: run(agent, calls, messages)), daemon=True)
retry_worker = None
retry_agent = None
worker.start()
try:
requests = [published.get(timeout=10) for _ in calls]
assert len({r["request_id"] for r in requests}) == len(calls)
assert approval.resolve_gateway_approval(key, "once", request_id=requests[1]["request_id"]) == 1
agent.interrupt("cancel while publishing")
worker.join(5)
assert not worker.is_alive()
assert errors == []
assert [row["tool_call_id"] for row in messages] == [c.id for c in calls]
assert approval.list_gateway_approvals(key) == []
assert not (tmp_path / "effect.txt").exists()
assert approval.resolve_gateway_approval(key, "once", request_id=requests[0]["request_id"]) == 0
release_notify.set()
# Same command AND call id in a later run still needs fresh consent.
retry_agent = _agent()
retry_agent._flush_messages_to_session_db = lambda *a, **kw: True
retry_rows = []
retry_worker = threading.Thread(target=propagate_context_to_thread(
lambda: run(retry_agent, [_call("second", command)], retry_rows)), daemon=True)
retry_worker.start()
request = published.get(timeout=10)
assert request["request_id"] not in {r["request_id"] for r in requests}
assert not (tmp_path / "effect.txt").exists()
assert approval.resolve_gateway_approval(key, "deny", request_id=request["request_id"]) == 1
retry_worker.join(5)
assert not retry_worker.is_alive()
assert errors == []
assert json.loads(retry_rows[0]["content"])["status"] == "blocked"
assert not (tmp_path / "effect.txt").exists()
assert approval.list_gateway_approvals(key) == []
finally:
release_notify.set()
agent.interrupt("test cleanup")
if retry_agent is not None:
retry_agent.interrupt("test cleanup")
approval.unregister_gateway_notify(key)
worker.join(5)
if retry_worker is not None:
retry_worker.join(5)
cleanup_vm(key)
clear_session_vars(tokens)
def test_failed_command_re_gates_later_prepared_approvals(tmp_path, monkeypatch):
"""#113158: the batch collects every approval before any command runs. When an
earlier command in the batch FAILS, the informed consent gathered for the later
ones describes a world that no longer holds: the prepared decision must be
discarded and the guard re-run live (a fresh approval request), instead of being
consumed as though nothing had failed."""
from tools.terminal_scope import reset_terminal_scope, set_terminal_scope
from tools.terminal_tool_lifecycle import cleanup_vm
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
monkeypatch.setenv("HERMES_EXEC_ASK", "1")
monkeypatch.setenv("TERMINAL_ENV", "local")
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
monkeypatch.setattr("tools.approval_context._get_approval_mode", lambda: "manual")
monkeypatch.setattr("tools.approval._tirith_scan", lambda command: {"action": "allow"})
monkeypatch.setattr("agent.title_generator.maybe_auto_title", lambda *a, **kw: None)
key = "regate-terminal-batch"
# First command fails (exit 1); second would succeed if allowed to run.
effect = tmp_path / "effect-later.txt"
commands = {"failing": "rm -rf absent-first; false",
"later": f"rm -rf absent-second; printf ran > '{effect}'"}
agent = _agent()
published = queue.Queue()
approval.register_gateway_notify(key, published.put)
from tui_gateway import server
monkeypatch.setattr(server, "_sessions", {key: {
"session_key": key, "source": "desktop", "agent": agent, "cwd": str(tmp_path),
}})
tokens = server._set_session_context(key)
calls = [_call(call_id, commands[call_id]) for call_id in ("failing", "later")]
executed = []
messages = []
errors = []
agent._flush_messages_to_session_db = lambda *a, **kw: True
def started(call_id, name, args):
executed.append((call_id, args["command"]))
agent.tool_start_callback = started
def run():
try:
agent._execute_tool_calls(SimpleNamespace(tool_calls=calls), messages, key)
except BaseException as exc:
errors.append(exc)
with ExitStack() as scope:
scope.callback(reset_terminal_scope, set_terminal_scope({"TERMINAL_ENV": "local", "TERMINAL_CWD": str(tmp_path)}))
worker = threading.Thread(target=propagate_context_to_thread(run), daemon=True)
worker.start()
try:
# Preparation collects BOTH approvals before any command runs.
first_request = published.get(timeout=10)
later_request = published.get(timeout=5)
assert {first_request["command"], later_request["command"]} == set(commands.values())
assert executed == []
assert approval.resolve_gateway_approval(key, "once", request_id=later_request["request_id"]) == 1
assert approval.resolve_gateway_approval(key, "once", request_id=first_request["request_id"]) == 1
# The failing command runs and its failure is published.
worker_join_marker = time.monotonic()
while not executed and time.monotonic() - worker_join_marker < 10:
time.sleep(0.05)
assert executed == [("failing", commands["failing"])]
# The later command must NOT run on the pre-collected approval: the
# guard re-runs and publishes a FRESH approval request, and the
# shell only starts once that one is answered.
regated = published.get(timeout=10)
assert regated["command"] == commands["later"]
assert regated["request_id"] != later_request["request_id"]
assert not effect.exists()
assert approval.resolve_gateway_approval(key, "once", request_id=regated["request_id"]) == 1
worker.join(timeout=15)
assert not worker.is_alive()
assert errors == []
assert executed == [("failing", commands["failing"]), ("later", commands["later"])]
assert [m["tool_call_id"] for m in messages] == [c.id for c in calls]
results = {m["tool_call_id"]: json.loads(m["content"]) for m in messages}
assert results["failing"]["exit_code"] == 1
assert results["later"]["exit_code"] == 0
assert effect.exists() and effect.read_text() == "ran"
assert approval.list_gateway_approvals(key) == []
finally:
agent.interrupt("test cleanup")
approval.unregister_gateway_notify(key)
worker.join(timeout=5)
cleanup_vm(key)
clear_session_vars(tokens)