219 lines
8.1 KiB
Python
219 lines
8.1 KiB
Python
"""Policy and bypass-regression coverage for kuri-skill-gate."""
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import time
|
|
import tomllib
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
PLUGIN_DIR = Path(__file__).resolve().parent.parent
|
|
PLUGIN_INIT = PLUGIN_DIR / "__init__.py"
|
|
|
|
ALLOWED_POLICY_FIXTURES = (
|
|
("read_file", {"path": "notes.md"}),
|
|
("terminal", {"command": "echo github"}),
|
|
)
|
|
HARD_BLOCK_POLICY_FIXTURES = (
|
|
("patch", {"path": "outside.py"}, "patch-tool-safety"),
|
|
("delegate_task", {"goal": "delegate"}, "delegate-task-best-practices"),
|
|
("terminal", {"command": "gh pr view"}, "gh-cli"),
|
|
)
|
|
BYPASS_REGRESSION_FIXTURES = (
|
|
"sh -c 'gh pr view'",
|
|
"$(gh pr view)",
|
|
"echo 'gh'",
|
|
)
|
|
|
|
|
|
def _load_mod():
|
|
spec = importlib.util.spec_from_file_location("kuri_skill_gate_evidence", PLUGIN_INIT)
|
|
assert spec is not None and spec.loader is not None
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
def _success_result() -> str:
|
|
return json.dumps({"success": True})
|
|
|
|
|
|
class TestPolicyFixtures(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mod: Any = _load_mod()
|
|
|
|
def test_plugin_manifest_and_project_versions_match(self):
|
|
manifest = (PLUGIN_DIR / "plugin.yaml").read_text(encoding="utf-8")
|
|
manifest_version = next(
|
|
line.split(":", maxsplit=1)[1].strip().strip('"')
|
|
for line in manifest.splitlines()
|
|
if line.startswith("version:")
|
|
)
|
|
project = tomllib.loads((PLUGIN_DIR / "pyproject.toml").read_text(encoding="utf-8"))
|
|
self.assertEqual(project["project"]["version"], manifest_version)
|
|
|
|
def test_allowed_policy_fixtures_pass_through(self):
|
|
for tool_name, args in ALLOWED_POLICY_FIXTURES:
|
|
with self.subTest(tool_name=tool_name, args=args):
|
|
next_call = MagicMock(return_value="allowed")
|
|
result = self.mod._on_tool_execution(
|
|
tool_name=tool_name,
|
|
args=args,
|
|
session_id="allowed",
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_called_once_with(args)
|
|
self.assertEqual(result, "allowed")
|
|
|
|
def test_hard_block_policy_fixtures_do_not_execute(self):
|
|
for tool_name, args, required_skill in HARD_BLOCK_POLICY_FIXTURES:
|
|
with self.subTest(tool_name=tool_name, args=args):
|
|
next_call = MagicMock(return_value="executed")
|
|
result = self.mod._on_tool_execution(
|
|
tool_name=tool_name,
|
|
args=args,
|
|
session_id="blocked",
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_not_called()
|
|
self.assertIn("blocked by kuri-skill-gate", result)
|
|
self.assertIn(required_skill, result)
|
|
|
|
def test_quoted_and_subshell_gh_bypasses_are_blocked(self):
|
|
for command in BYPASS_REGRESSION_FIXTURES:
|
|
with self.subTest(command=command):
|
|
next_call = MagicMock(return_value="executed")
|
|
result = self.mod._on_tool_execution(
|
|
tool_name="terminal",
|
|
args={"command": command},
|
|
session_id="quoted-gh",
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_not_called()
|
|
self.assertIn("gh-cli", result)
|
|
|
|
|
|
class TestSkillReadEvidence(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mod: Any = _load_mod()
|
|
|
|
def _view(self, args, session_id="skill-read", result=None):
|
|
next_call = MagicMock(return_value=_success_result() if result is None else result)
|
|
observed = self.mod._on_tool_execution(
|
|
tool_name="skill_view",
|
|
args=args,
|
|
session_id=session_id,
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_called_once_with(args)
|
|
return observed
|
|
|
|
def _assert_patch_blocked(self, session_id="skill-read"):
|
|
next_call = MagicMock(return_value="patched")
|
|
result = self.mod._on_tool_execution(
|
|
tool_name="patch",
|
|
args={"path": "outside.py"},
|
|
session_id=session_id,
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_not_called()
|
|
self.assertIn("patch-tool-safety", result)
|
|
|
|
def test_successful_main_skill_view_unlocks(self):
|
|
self._view({"name": "patch-tool-safety"})
|
|
next_call = MagicMock(return_value="patched")
|
|
result = self.mod._on_tool_execution(
|
|
tool_name="patch",
|
|
args={"path": "outside.py"},
|
|
session_id="skill-read",
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_called_once()
|
|
self.assertEqual(result, "patched")
|
|
|
|
def test_failed_skill_view_does_not_unlock(self):
|
|
self._view({"name": "patch-tool-safety"}, result=json.dumps({"success": False}))
|
|
self._assert_patch_blocked()
|
|
|
|
def test_reference_file_skill_view_does_not_unlock(self):
|
|
self._view({"name": "patch-tool-safety", "file_path": "references/guide.md"})
|
|
self._assert_patch_blocked()
|
|
|
|
def test_missing_next_call_does_not_unlock(self):
|
|
self.mod._on_tool_execution(
|
|
tool_name="skill_view",
|
|
args={"name": "patch-tool-safety"},
|
|
session_id="no-result",
|
|
next_call=None,
|
|
)
|
|
self._assert_patch_blocked("no-result")
|
|
|
|
|
|
class TestSelfGateContainment(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mod: Any = _load_mod()
|
|
|
|
def test_existing_real_path_inside_plugin_is_allowed(self):
|
|
next_call = MagicMock(return_value="patched")
|
|
target = str(PLUGIN_INIT)
|
|
result = self.mod._on_tool_execution(
|
|
tool_name="patch",
|
|
args={"path": target},
|
|
session_id="self",
|
|
next_call=next_call,
|
|
)
|
|
next_call.assert_called_once()
|
|
self.assertEqual(result, "patched")
|
|
|
|
def test_nonexistent_or_prefix_or_symlink_escape_is_blocked(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
root = Path(temp_dir)
|
|
plugin_dir = root / "plugin"
|
|
plugin_dir.mkdir()
|
|
inside = plugin_dir / "inside.py"
|
|
inside.write_text("inside", encoding="utf-8")
|
|
prefix_dir = root / "plugin-other"
|
|
prefix_dir.mkdir()
|
|
outside = prefix_dir / "outside.py"
|
|
outside.write_text("outside", encoding="utf-8")
|
|
link = plugin_dir / "outside-link.py"
|
|
link.symlink_to(outside)
|
|
|
|
original_plugin_dir = self.mod._PLUGIN_DIR
|
|
self.mod._PLUGIN_DIR = os.path.realpath(plugin_dir)
|
|
try:
|
|
self.assertTrue(self.mod._is_existing_self_target({"path": str(inside)}))
|
|
self.assertFalse(
|
|
self.mod._is_existing_self_target({"path": str(prefix_dir / "missing.py")})
|
|
)
|
|
self.assertFalse(self.mod._is_existing_self_target({"path": str(outside)}))
|
|
self.assertFalse(self.mod._is_existing_self_target({"path": str(link)}))
|
|
finally:
|
|
self.mod._PLUGIN_DIR = original_plugin_dir
|
|
|
|
|
|
class TestLoadedSkillStateBounds(unittest.TestCase):
|
|
def setUp(self):
|
|
self.mod: Any = _load_mod()
|
|
|
|
def test_expired_sessions_are_lazily_removed(self):
|
|
self.mod._loaded_skills["expired"] = self.mod._SessionSkillState(
|
|
{"patch-tool-safety"}, time.monotonic() - self.mod._SESSION_TTL_SECONDS - 1
|
|
)
|
|
self.assertFalse(self.mod._has_loaded_skill("expired", "patch-tool-safety"))
|
|
self.assertNotIn("expired", self.mod._loaded_skills)
|
|
|
|
def test_session_state_is_capped_at_1024(self):
|
|
for index in range(self.mod._MAX_TRACKED_SESSIONS + 2):
|
|
self.mod._record_successful_skill_read(
|
|
f"session-{index}",
|
|
{"name": "patch-tool-safety"},
|
|
_success_result(),
|
|
)
|
|
self.assertEqual(len(self.mod._loaded_skills), self.mod._MAX_TRACKED_SESSIONS)
|
|
self.assertNotIn("session-0", self.mod._loaded_skills)
|
|
self.assertIn("session-1025", self.mod._loaded_skills)
|