kuri-skill-gate/tests/test_gate_evidence.py

219 lines
8.1 KiB
Python

"""Policy and bypass-regression coverage for kuri-skill-gate."""
import importlib.util
import json
import os
import tempfile
import time
import tomllib
import unittest
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock
PLUGIN_DIR = Path(__file__).resolve().parent.parent
PLUGIN_INIT = PLUGIN_DIR / "__init__.py"
ALLOWED_POLICY_FIXTURES = (
("read_file", {"path": "notes.md"}),
("terminal", {"command": "echo github"}),
)
HARD_BLOCK_POLICY_FIXTURES = (
("patch", {"path": "outside.py"}, "patch-tool-safety"),
("delegate_task", {"goal": "delegate"}, "delegate-task-best-practices"),
("terminal", {"command": "gh pr view"}, "gh-cli"),
)
BYPASS_REGRESSION_FIXTURES = (
"sh -c 'gh pr view'",
"$(gh pr view)",
"echo 'gh'",
)
def _load_mod():
spec = importlib.util.spec_from_file_location("kuri_skill_gate_evidence", PLUGIN_INIT)
assert spec is not None and spec.loader is not None
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _success_result() -> str:
return json.dumps({"success": True})
class TestPolicyFixtures(unittest.TestCase):
def setUp(self):
self.mod: Any = _load_mod()
def test_plugin_manifest_and_project_versions_match(self):
manifest = (PLUGIN_DIR / "plugin.yaml").read_text(encoding="utf-8")
manifest_version = next(
line.split(":", maxsplit=1)[1].strip().strip('"')
for line in manifest.splitlines()
if line.startswith("version:")
)
project = tomllib.loads((PLUGIN_DIR / "pyproject.toml").read_text(encoding="utf-8"))
self.assertEqual(project["project"]["version"], manifest_version)
def test_allowed_policy_fixtures_pass_through(self):
for tool_name, args in ALLOWED_POLICY_FIXTURES:
with self.subTest(tool_name=tool_name, args=args):
next_call = MagicMock(return_value="allowed")
result = self.mod._on_tool_execution(
tool_name=tool_name,
args=args,
session_id="allowed",
next_call=next_call,
)
next_call.assert_called_once_with(args)
self.assertEqual(result, "allowed")
def test_hard_block_policy_fixtures_do_not_execute(self):
for tool_name, args, required_skill in HARD_BLOCK_POLICY_FIXTURES:
with self.subTest(tool_name=tool_name, args=args):
next_call = MagicMock(return_value="executed")
result = self.mod._on_tool_execution(
tool_name=tool_name,
args=args,
session_id="blocked",
next_call=next_call,
)
next_call.assert_not_called()
self.assertIn("blocked by kuri-skill-gate", result)
self.assertIn(required_skill, result)
def test_quoted_and_subshell_gh_bypasses_are_blocked(self):
for command in BYPASS_REGRESSION_FIXTURES:
with self.subTest(command=command):
next_call = MagicMock(return_value="executed")
result = self.mod._on_tool_execution(
tool_name="terminal",
args={"command": command},
session_id="quoted-gh",
next_call=next_call,
)
next_call.assert_not_called()
self.assertIn("gh-cli", result)
class TestSkillReadEvidence(unittest.TestCase):
def setUp(self):
self.mod: Any = _load_mod()
def _view(self, args, session_id="skill-read", result=None):
next_call = MagicMock(return_value=_success_result() if result is None else result)
observed = self.mod._on_tool_execution(
tool_name="skill_view",
args=args,
session_id=session_id,
next_call=next_call,
)
next_call.assert_called_once_with(args)
return observed
def _assert_patch_blocked(self, session_id="skill-read"):
next_call = MagicMock(return_value="patched")
result = self.mod._on_tool_execution(
tool_name="patch",
args={"path": "outside.py"},
session_id=session_id,
next_call=next_call,
)
next_call.assert_not_called()
self.assertIn("patch-tool-safety", result)
def test_successful_main_skill_view_unlocks(self):
self._view({"name": "patch-tool-safety"})
next_call = MagicMock(return_value="patched")
result = self.mod._on_tool_execution(
tool_name="patch",
args={"path": "outside.py"},
session_id="skill-read",
next_call=next_call,
)
next_call.assert_called_once()
self.assertEqual(result, "patched")
def test_failed_skill_view_does_not_unlock(self):
self._view({"name": "patch-tool-safety"}, result=json.dumps({"success": False}))
self._assert_patch_blocked()
def test_reference_file_skill_view_does_not_unlock(self):
self._view({"name": "patch-tool-safety", "file_path": "references/guide.md"})
self._assert_patch_blocked()
def test_missing_next_call_does_not_unlock(self):
self.mod._on_tool_execution(
tool_name="skill_view",
args={"name": "patch-tool-safety"},
session_id="no-result",
next_call=None,
)
self._assert_patch_blocked("no-result")
class TestSelfGateContainment(unittest.TestCase):
def setUp(self):
self.mod: Any = _load_mod()
def test_existing_real_path_inside_plugin_is_allowed(self):
next_call = MagicMock(return_value="patched")
target = str(PLUGIN_INIT)
result = self.mod._on_tool_execution(
tool_name="patch",
args={"path": target},
session_id="self",
next_call=next_call,
)
next_call.assert_called_once()
self.assertEqual(result, "patched")
def test_nonexistent_or_prefix_or_symlink_escape_is_blocked(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
plugin_dir = root / "plugin"
plugin_dir.mkdir()
inside = plugin_dir / "inside.py"
inside.write_text("inside", encoding="utf-8")
prefix_dir = root / "plugin-other"
prefix_dir.mkdir()
outside = prefix_dir / "outside.py"
outside.write_text("outside", encoding="utf-8")
link = plugin_dir / "outside-link.py"
link.symlink_to(outside)
original_plugin_dir = self.mod._PLUGIN_DIR
self.mod._PLUGIN_DIR = os.path.realpath(plugin_dir)
try:
self.assertTrue(self.mod._is_existing_self_target({"path": str(inside)}))
self.assertFalse(
self.mod._is_existing_self_target({"path": str(prefix_dir / "missing.py")})
)
self.assertFalse(self.mod._is_existing_self_target({"path": str(outside)}))
self.assertFalse(self.mod._is_existing_self_target({"path": str(link)}))
finally:
self.mod._PLUGIN_DIR = original_plugin_dir
class TestLoadedSkillStateBounds(unittest.TestCase):
def setUp(self):
self.mod: Any = _load_mod()
def test_expired_sessions_are_lazily_removed(self):
self.mod._loaded_skills["expired"] = self.mod._SessionSkillState(
{"patch-tool-safety"}, time.monotonic() - self.mod._SESSION_TTL_SECONDS - 1
)
self.assertFalse(self.mod._has_loaded_skill("expired", "patch-tool-safety"))
self.assertNotIn("expired", self.mod._loaded_skills)
def test_session_state_is_capped_at_1024(self):
for index in range(self.mod._MAX_TRACKED_SESSIONS + 2):
self.mod._record_successful_skill_read(
f"session-{index}",
{"name": "patch-tool-safety"},
_success_result(),
)
self.assertEqual(len(self.mod._loaded_skills), self.mod._MAX_TRACKED_SESSIONS)
self.assertNotIn("session-0", self.mod._loaded_skills)
self.assertIn("session-1025", self.mod._loaded_skills)