mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
# Conflicts: # apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts # apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts # apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts # apps/desktop/e2e/bot-mode-roster-localized.spec.ts # apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts # apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts # apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts # apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts # apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts # apps/desktop/e2e/bot-roster-user-sections.spec.ts # apps/desktop/e2e/bot-routines-pane-narrow.spec.ts # apps/desktop/e2e/bot-row-open-recent-session.spec.ts # apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts # apps/desktop/e2e/group-composer-auto-grow.spec.ts # apps/desktop/e2e/group-create-gate-remote-roster.spec.ts # apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts # apps/desktop/e2e/hosted-room-backend-continuity.spec.ts # apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts # apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts # apps/desktop/e2e/worktree-branch-status.spec.ts # apps/desktop/electron/backend-probes.test.ts # apps/desktop/electron/connection-apply.test.ts # apps/desktop/electron/desktop-electron-pin.test.ts # apps/desktop/electron/desktop-uninstall.test.ts # apps/desktop/electron/gateway-file-download-transport.test.ts # apps/desktop/electron/gateway-stop-before-update.test.ts # apps/desktop/electron/github-api-auth.test.ts # apps/desktop/electron/registry-primary-profile-scope.test.ts # apps/desktop/electron/update-api-check.test.ts # apps/desktop/electron/update-handoff-marker.test.ts # apps/desktop/electron/venv-blocker-scan.test.ts # apps/desktop/scripts/after-extract.test.mjs # apps/desktop/scripts/local-pack-publish.test.mjs # apps/desktop/scripts/tasks-scroll.test.mjs # apps/desktop/src/app/settings/model-settings.test.tsx # apps/desktop/src/app/updates-overlay.blockers.test.tsx # apps/desktop/src/components/desktop-install-overlay.test.tsx # apps/desktop/src/lib/update-copy.test.ts # scripts/ci/check_os_marker_fakes.py # tests-js/desktop-mac-usage-descriptions.test.ts # tests-js/node-engine-alignment.test.ts # tests/agent/lsp/test_install_and_lint_fixes.py # tests/agent/test_command_token_source.py # tests/agent/test_compression_boundary_hook.py # tests/agent/test_create_openai_client_ssl_verify.py # tests/agent/test_custom_provider_ca_probes.py # tests/agent/test_endpoint_blackhole.py # tests/agent/test_estimator_parity.py # tests/agent/test_in_place_compaction.py # tests/agent/test_moa_loop_mode.py # tests/agent/test_model_metadata.py # tests/agent/test_skill_session_platform_gate.py # tests/agent/test_skill_utils.py # tests/agent/test_ssl_ca_guard.py # tests/computer_use/test_doctor.py # tests/cron/test_codex_execution_paths.py # tests/cron/test_cron_bot_chat_delivery.py # tests/cron/test_cron_script.py # tests/cron/test_media_delivery_parity.py # tests/cron/test_misfire_catchup.py # tests/cron/test_parallel_pool.py # tests/cron/test_recurring_eagain_redispatch.py # tests/gateway/test_choice_picker.py # tests/gateway/test_control_socket_windows_live.py # tests/gateway/test_dingtalk.py # tests/gateway/test_feishu.py # tests/gateway/test_feishu_onboard.py # tests/gateway/test_gateway_shutdown.py # tests/gateway/test_matrix.py # tests/gateway/test_model_command_custom_providers.py # tests/gateway/test_reasoning_command.py # tests/gateway/test_runtime_footer.py # tests/gateway/test_session.py # tests/gateway/test_session_hygiene.py # tests/gateway/test_status.py # tests/gateway/test_teams.py # tests/gateway/test_turn_lease.py # tests/gateway/test_whatsapp_connect.py # tests/hermes_cli/test_approvals_command.py # tests/hermes_cli/test_auth_store_lock_concurrent.py # tests/hermes_cli/test_backup.py # tests/hermes_cli/test_banner_git_state.py # tests/hermes_cli/test_certifi_repair.py # tests/hermes_cli/test_cmd_update.py # tests/hermes_cli/test_compat_manifest_targets.py # tests/hermes_cli/test_computer_use_cli.py # tests/hermes_cli/test_cpr_local_leak.py # tests/hermes_cli/test_dashboard_auth_gate.py # tests/hermes_cli/test_dashboard_procs_kill_grace.py # tests/hermes_cli/test_desktop_lifecycle_windows_live.py # tests/hermes_cli/test_doctor.py # tests/hermes_cli/test_doctor_command_install.py # tests/hermes_cli/test_fleet_config_migration_windows_live.py # tests/hermes_cli/test_gateway.py # tests/hermes_cli/test_gateway_platform_gating.py # tests/hermes_cli/test_gateway_restart_loop.py # tests/hermes_cli/test_gateway_task_probe.py # tests/hermes_cli/test_gateway_wsl.py # tests/hermes_cli/test_gui_command.py # tests/hermes_cli/test_install_cua_driver.py # tests/hermes_cli/test_kanban_db.py # tests/hermes_cli/test_lazy_command_exports.py # tests/hermes_cli/test_lazy_refresh_venv_repair.py # tests/hermes_cli/test_linux_desktop_entry.py # tests/hermes_cli/test_local_runtime.py # tests/hermes_cli/test_local_runtime_updates.py # tests/hermes_cli/test_managed_uv.py # tests/hermes_cli/test_mcp_reload_confirm_gate.py # tests/hermes_cli/test_nous_subscription.py # tests/hermes_cli/test_npm_engine.py # tests/hermes_cli/test_personality_none.py # tests/hermes_cli/test_pet_toggle.py # tests/hermes_cli/test_plan_reconciliation_windows_live.py # tests/hermes_cli/test_plugin_event_bus.py # tests/hermes_cli/test_plugin_manifest_v2.py # tests/hermes_cli/test_plugin_packs.py # tests/hermes_cli/test_plugins_cmd.py # tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py # tests/hermes_cli/test_process_identity.py # tests/hermes_cli/test_profiles.py # tests/hermes_cli/test_profiles_sidebar_cache.py # tests/hermes_cli/test_pty_bridge.py # tests/hermes_cli/test_resolve_turn_limit.py # tests/hermes_cli/test_serve_runtime_inventory.py # tests/hermes_cli/test_session_vacuum_config.py # tests/hermes_cli/test_set_config_value.py # tests/hermes_cli/test_signal_handler_kanban_worker.py # tests/hermes_cli/test_slash_confirm_windows.py # tests/hermes_cli/test_stale_pid_guard.py # tests/hermes_cli/test_startup_fast_guards.py # tests/hermes_cli/test_status.py # tests/hermes_cli/test_telegram_managed_bot.py # tests/hermes_cli/test_tools_config.py # tests/hermes_cli/test_update_apply_shallow_count.py # tests/hermes_cli/test_update_autostash.py # tests/hermes_cli/test_update_concurrent_quarantine.py # tests/hermes_cli/test_update_fetch_failure_classifier.py # tests/hermes_cli/test_update_fleet_probe_resume_token.py # tests/hermes_cli/test_update_handoff_backend_reap.py # tests/hermes_cli/test_update_handoff_desktop_rebuild.py # tests/hermes_cli/test_update_head_moved_gate.py # tests/hermes_cli/test_update_host_obligation.py # tests/hermes_cli/test_update_import_guard.py # tests/hermes_cli/test_update_interrupted_recovery.py # tests/hermes_cli/test_update_inventory.py # tests/hermes_cli/test_update_launchd_unloaded_gateway.py # tests/hermes_cli/test_update_missing_configured_deps.py # tests/hermes_cli/test_update_modified_notice.py # tests/hermes_cli/test_update_multiplex_migration_hook.py # tests/hermes_cli/test_update_no_gateway_restart.py # tests/hermes_cli/test_update_orphan_backend_reap.py # tests/hermes_cli/test_update_parked_branch_guard.py # tests/hermes_cli/test_update_post_pull_syntax_guard.py # tests/hermes_cli/test_update_receipt.py # tests/hermes_cli/test_update_self_lock.py # tests/hermes_cli/test_update_shim_fail_closed.py # tests/hermes_cli/test_update_shim_self_lock.py # tests/hermes_cli/test_update_sqlite_remediation.py # tests/hermes_cli/test_update_stale_dashboard.py # tests/hermes_cli/test_update_stale_virtualenv.py # tests/hermes_cli/test_update_venv_health.py # tests/hermes_cli/test_update_venv_ownership_preflight.py # tests/hermes_cli/test_update_wedged_gateway.py # tests/hermes_cli/test_update_yes_flag.py # tests/hermes_cli/test_update_zip_two_phase.py # tests/hermes_cli/test_urllib_security.py # tests/hermes_cli/test_ux_messages_auth_config.py # tests/hermes_cli/test_ux_messages_startup.py # tests/hermes_cli/test_venv_holder_classifier.py # tests/hermes_cli/test_verify_console_scripts.py # tests/hermes_cli/test_verify_core_dependencies.py # tests/hermes_cli/test_web_server.py # tests/hermes_cli/test_web_server_console_ws.py # tests/hermes_cli/test_web_server_ws_ping.py # tests/hermes_cli/test_web_ui_build.py # tests/hermes_state/test_fts_rebuild_admission.py # tests/hermes_state/test_hermes_state.py # tests/plugins/memory/test_memory_lazy_install.py # tests/plugins/test_google_meet_plugin.py # tests/plugins/test_langfuse_plugin.py # tests/plugins/test_security_guidance_plugin.py # tests/plugins/test_transform_llm_output_hook.py # tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py # tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py # tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py # tests/scripts/install/test_install_clone_throttle_fallback.py # tests/scripts/install/test_install_lockfile_churn.py # tests/scripts/install/test_install_no_initial_commit.py # tests/scripts/install/test_install_sh_browser_install.py # tests/scripts/install/test_install_sh_node_prerelease.py # tests/scripts/install/test_install_sh_symlink_stomp.py # tests/scripts/install/test_install_sh_uv_lock_config.py # tests/scripts/install/test_install_unmerged_index.py # tests/scripts/test_contributor_map.py # tests/scripts/test_run_tests_parallel.py # tests/skills/test_competitor_news_monitor_skill.py # tests/skills/test_document_to_action_items_skill.py # tests/skills/test_google_workspace_setup.py # tests/skills/test_google_workspace_setup_deps.py # tests/skills/test_grounded_citations_skill.py # tests/skills/test_ip_as_logo_skill.py # tests/skills/test_live_dashboard_skill.py # tests/skills/test_mcp_oauth_remote_gateway_skill.py # tests/skills/test_office_document_skills.py # tests/skills/test_openclaw_migration.py # tests/skills/test_product_price_monitor_skill.py # tests/skills/test_scrollcraft_skill.py # tests/skills/test_setup_wizard_generator_skill.py # tests/skills/test_weekly_review_planning_skill.py # tests/test_engines_satisfiable.py # tests/test_fast_safe_load.py # tests/test_hermes_bootstrap.py # tests/test_hermes_constants.py # tests/test_hermes_logging.py # tests/test_managed_runtime_resolution.py # tests/test_model_tools_async_bridge.py # tests/test_packaging_build_guard.py # tests/test_packaging_metadata.py # tests/test_yaml_indent_consistency.py # tests/tools/test_approval_timeout_overflow.py # tests/tools/test_base_environment.py # tests/tools/test_bot_mode_dm.py # tests/tools/test_browser_chromium_check.py # tests/tools/test_browser_hardening.py # tests/tools/test_browser_homebrew_paths.py # tests/tools/test_browser_npx_warmup.py # tests/tools/test_browser_orphan_reaper.py # tests/tools/test_browser_real_profile.py # tests/tools/test_browser_use_cli.py # tests/tools/test_clipboard.py # tests/tools/test_code_execution.py # tests/tools/test_code_execution_modes.py # tests/tools/test_code_execution_windows_env.py # tests/tools/test_computer_use.py # tests/tools/test_delegate_liveness_timeout.py # tests/tools/test_execute_code_approval_cluster.py # tests/tools/test_execution_flag_detection.py # tests/tools/test_fal_common.py # tests/tools/test_file_operations.py # tests/tools/test_file_tools.py # tests/tools/test_file_tools_cwd_resolution.py # tests/tools/test_file_tools_live.py # tests/tools/test_lazy_deps.py # tests/tools/test_lazy_deps_durable_target.py # tests/tools/test_lazy_deps_managed.py # tests/tools/test_local_env_blocklist.py # tests/tools/test_local_tempdir.py # tests/tools/test_macos_protected_search.py # tests/tools/test_mcp_npx_cached_bin.py # tests/tools/test_oneshot_completion_linger.py # tests/tools/test_process_registry.py # tests/tools/test_read_file_schema_gating.py # tests/tools/test_skill_improvements.py # tests/tools/test_skills_sync.py # tests/tools/test_termux_api_detection.py # tests/tools/test_tirith_security.py # tests/tools/test_transcription_tools.py # tests/tools/test_tts_streaming.py # tests/tools/test_wake_word.py # tests/tui_gateway/test_compute_host_borrowed_lease.py # tests/tui_gateway/test_compute_host_turn_protocol.py # tests/tui_gateway/test_isolated_orphan_activity.py # tests/tui_gateway/test_protocol.py # tests/tui_gateway/test_slash_worker_profile_home.py # tests/tui_gateway/test_subprocess_encoding.py # tests/tui_gateway/test_tui_gateway_server.py # ui-tui/src/__tests__/terminalParity.test.ts # ui-tui/src/__tests__/termuxComposerLayout.test.ts # ui-tui/src/__tests__/textInputFastEcho.test.ts
659 lines
29 KiB
Python
659 lines
29 KiB
Python
"""Tests for the WAL→DELETE journal-mode fallback on NFS / SMB / FUSE.
|
|
|
|
When ``PRAGMA journal_mode=WAL`` raises ``OperationalError("locking protocol")``
|
|
(SQLITE_PROTOCOL — typical on NFS/SMB), Hermes must fall back to
|
|
``journal_mode=DELETE`` so ``state.db`` / ``kanban.db`` remain usable.
|
|
|
|
Without this fallback, users on NFS-mounted ``HERMES_HOME`` silently lose
|
|
``/resume``, ``/title``, ``/history``, ``/branch``, session search, and the
|
|
kanban dispatcher — because ``SessionDB()`` init propagates the error and
|
|
every caller swallows it, leaving ``_session_db = None``.
|
|
|
|
See: https://www.sqlite.org/wal.html — "WAL does not work over a network
|
|
filesystem".
|
|
"""
|
|
|
|
import sqlite3
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
import hermes_state
|
|
import hermes_state_wal
|
|
from hermes_state import SessionDB, get_last_init_error
|
|
from hermes_state_wal import WalUnsupportedError, apply_wal_with_fallback
|
|
|
|
|
|
# ``sqlite3.Connection.execute`` is a C-level slot and can't be monkeypatched
|
|
# directly (``'sqlite3.Connection' object attribute 'execute' is read-only``).
|
|
# A factory-built subclass lets us intercept journal_mode=WAL per-test with
|
|
# its own mutable counter, avoiding a parallel-run class-state race.
|
|
def _make_blocking_factory(reason: str, attempt_counter: list):
|
|
"""Return a sqlite3.Connection subclass that raises on PRAGMA journal_mode=WAL."""
|
|
|
|
class _WalBlockingConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
if "journal_mode=wal" in sql.lower().replace(" ", ""):
|
|
attempt_counter[0] += 1
|
|
raise sqlite3.OperationalError(reason)
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
return _WalBlockingConnection
|
|
|
|
|
|
def _open_blocking(path, reason="locking protocol", **kwargs):
|
|
"""Open a connection whose WAL pragma raises ``reason``.
|
|
|
|
Returns ``(conn, attempt_counter_list)`` so callers can assert how many
|
|
times WAL was attempted.
|
|
"""
|
|
attempts = [0]
|
|
factory = _make_blocking_factory(reason, attempts)
|
|
return sqlite3.connect(str(path), factory=factory, **kwargs), attempts
|
|
|
|
|
|
def _make_silent_noop_factory(returned_mode: str = "delete"):
|
|
"""Return a ``sqlite3.Connection`` subclass whose ``PRAGMA journal_mode=WAL``
|
|
silently NO-OPs: it returns the still-effective journal mode (e.g.
|
|
``delete``) WITHOUT raising — the way macOS NFS / SMB / the AgentFS NFS
|
|
overlay behave. NFS that *raises* SQLITE_PROTOCOL is covered by
|
|
:func:`_make_blocking_factory`; this is the other, quieter failure shape.
|
|
"""
|
|
|
|
class _WalSilentNoOpConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
if "journal_mode=wal" in sql.lower().replace(" ", ""):
|
|
# Refuse the WAL switch but DON'T raise; report the mode that is
|
|
# actually still in force, exactly as the NFS client does.
|
|
return super().execute(
|
|
f"PRAGMA journal_mode={returned_mode}", *args, **kwargs
|
|
)
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
return _WalSilentNoOpConnection
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_last_init_error():
|
|
"""Reset the module-global last-error before and after each test."""
|
|
hermes_state._set_last_init_error(None)
|
|
yield
|
|
hermes_state._set_last_init_error(None)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_wal_fallback_warned_paths():
|
|
"""Reset the WAL-fallback warned-paths set so dedup doesn't leak between tests."""
|
|
hermes_state_wal._wal_fallback_warned_paths.clear()
|
|
hermes_state_wal._wal_delete_fallback_failed_paths.clear()
|
|
yield
|
|
hermes_state_wal._wal_fallback_warned_paths.clear()
|
|
hermes_state_wal._wal_delete_fallback_failed_paths.clear()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _assume_fixed_sqlite(monkeypatch):
|
|
"""NFS-fallback tests assume a SQLite build without the WAL-reset bug."""
|
|
monkeypatch.setattr(
|
|
hermes_state_wal, "is_sqlite_wal_reset_vulnerable", lambda version_info=None: False
|
|
)
|
|
hermes_state_wal._wal_reset_bug_warned_paths.clear()
|
|
yield
|
|
hermes_state_wal._wal_reset_bug_warned_paths.clear()
|
|
|
|
|
|
class TestApplyWalWithFallback:
|
|
def test_succeeds_on_local_fs(self, tmp_path):
|
|
"""Happy path: WAL works on a normal filesystem."""
|
|
conn = sqlite3.connect(str(tmp_path / "ok.db"), isolation_level=None)
|
|
mode = apply_wal_with_fallback(conn)
|
|
assert mode == "wal"
|
|
cur = conn.execute("PRAGMA journal_mode")
|
|
assert cur.fetchone()[0].lower() == "wal"
|
|
conn.close()
|
|
|
|
def test_falls_back_to_delete_on_locking_protocol(self, tmp_path, caplog):
|
|
"""NFS-style ``locking protocol`` error → DELETE mode + one ERROR."""
|
|
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
mode = apply_wal_with_fallback(conn, db_label="test.db")
|
|
|
|
assert mode == "delete"
|
|
errors = [r for r in caplog.records if r.levelname == "ERROR"]
|
|
assert len(errors) == 1
|
|
msg = errors[0].getMessage()
|
|
assert "test.db" in msg
|
|
assert "journal_mode=DELETE" in msg
|
|
assert "locking protocol" in msg
|
|
|
|
# Post-fallback the DB is still usable for real writes
|
|
conn.execute("CREATE TABLE t (x INTEGER)")
|
|
conn.execute("INSERT INTO t VALUES (1)")
|
|
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
|
|
conn.close()
|
|
|
|
def test_falls_back_on_not_authorized(self, tmp_path):
|
|
"""Some FUSE mounts block WAL pragma outright ('not authorized')."""
|
|
conn, _ = _open_blocking(
|
|
tmp_path / "fuse.db", reason="not authorized", isolation_level=None
|
|
)
|
|
mode = apply_wal_with_fallback(conn)
|
|
assert mode == "delete"
|
|
conn.close()
|
|
|
|
def test_falls_back_when_wal_silently_refused(self, tmp_path, caplog):
|
|
"""macOS NFS / SMB / AgentFS-NFS can REFUSE the WAL switch WITHOUT
|
|
raising: ``PRAGMA journal_mode=WAL`` returns the still-effective mode
|
|
('delete') and no exception. The marker-exception path never fires, so
|
|
the function must detect the silent no-op from the PRAGMA's RETURN
|
|
value — otherwise it returns a false 'wal' and skips the WARNING.
|
|
|
|
Reproduced on a real AgentFS NFS overlay, where
|
|
``PRAGMA journal_mode=WAL`` returned ('delete',) with no
|
|
OperationalError.
|
|
"""
|
|
factory = _make_silent_noop_factory("delete")
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "macnfs.db"), factory=factory, isolation_level=None
|
|
)
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
mode = apply_wal_with_fallback(conn, db_label="kanban.db")
|
|
|
|
assert mode == "delete", "must report the true mode, not a false 'wal'"
|
|
assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "delete"
|
|
errors = [r for r in caplog.records if r.levelname == "ERROR"]
|
|
assert len(errors) == 1, "silent no-op must still emit exactly one ERROR"
|
|
assert "kanban.db" in errors[0].getMessage()
|
|
conn.close()
|
|
|
|
def test_transient_disk_io_error_recovers_to_wal(self, tmp_path):
|
|
"""Transient EIO from ``PRAGMA journal_mode=WAL`` must NOT silently
|
|
downgrade to DELETE.
|
|
|
|
Regression for "Bug D": treating transient EIO as a permanent
|
|
WAL-incompat marker produced the mixed-journal-mode-across-processes
|
|
corruption pattern (process A downgrades to DELETE, sibling
|
|
processes successfully set WAL, SQLite corrupts the file because
|
|
the two locking protocols are documented as incompatible). EIO is
|
|
often transient (page-cache pressure, lock contention, brief
|
|
storage hiccups); the fallback retries the pragma, and a transient
|
|
EIO clears — returning "wal", never walking the DB into a
|
|
permanently downgraded state.
|
|
"""
|
|
attempts = [0]
|
|
|
|
class _TransientEIOConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
if "journal_mode=wal" in sql.lower().replace(" ", ""):
|
|
attempts[0] += 1
|
|
if attempts[0] == 1:
|
|
raise sqlite3.OperationalError("disk I/O error")
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "flaky.db"),
|
|
factory=_TransientEIOConnection,
|
|
isolation_level=None,
|
|
)
|
|
assert apply_wal_with_fallback(conn) == "wal"
|
|
assert attempts[0] == 2, "the retry must have re-attempted the pragma"
|
|
assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
|
conn.close()
|
|
|
|
def test_persistent_disk_io_error_falls_back_to_delete(self, tmp_path, caplog):
|
|
"""Deterministic EIO (ZFS / APFS-CoW SHM corruption — #55305, #71498)
|
|
keeps failing across retries → guarded fallback to DELETE with one
|
|
ERROR, instead of wedging the session DB entirely.
|
|
"""
|
|
conn, attempts = _open_blocking(
|
|
tmp_path / "zfs.db", reason="disk I/O error", isolation_level=None
|
|
)
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
mode = apply_wal_with_fallback(conn, db_label="zfs.db")
|
|
assert mode == "delete"
|
|
assert attempts[0] >= 3, "must retry before concluding EIO is persistent"
|
|
errors = [
|
|
r
|
|
for r in caplog.records
|
|
if r.levelname == "ERROR" and "zfs.db" in r.getMessage()
|
|
]
|
|
assert len(errors) == 1
|
|
# Post-fallback the DB is still usable for real writes
|
|
conn.execute("CREATE TABLE t (x INTEGER)")
|
|
conn.execute("INSERT INTO t VALUES (1)")
|
|
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
|
|
conn.close()
|
|
|
|
def test_persistent_disk_io_error_never_downgrades_wal_disk(self, tmp_path):
|
|
"""Persistent EIO on a DB whose on-disk header already says WAL must
|
|
re-raise (Bug D guard) — never flip a WAL file to DELETE under
|
|
possible concurrent openers.
|
|
"""
|
|
primer = sqlite3.connect(str(tmp_path / "waldisk.db"), isolation_level=None)
|
|
try:
|
|
primer.execute("PRAGMA journal_mode=WAL")
|
|
primer.execute("CREATE TABLE t (x INTEGER)")
|
|
finally:
|
|
primer.close()
|
|
|
|
class _EIOWalOnlyConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
normalized = sql.lower().replace(" ", "")
|
|
if "journal_mode=wal" in normalized:
|
|
raise sqlite3.OperationalError("disk I/O error")
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "waldisk.db"),
|
|
factory=_EIOWalOnlyConnection,
|
|
isolation_level=None,
|
|
)
|
|
# The read-only probe sees WAL and returns early on this shape; force
|
|
# the set-pragma path by making the probe report a non-WAL mode.
|
|
# (On-disk header still says WAL, which is what the guard checks.)
|
|
try:
|
|
result = apply_wal_with_fallback(conn)
|
|
# Probe short-circuit: acceptable, DB stays WAL.
|
|
assert result == "wal"
|
|
except sqlite3.OperationalError:
|
|
pass # re-raise path: equally acceptable — no downgrade happened
|
|
finally:
|
|
conn.close()
|
|
|
|
check = sqlite3.connect(str(tmp_path / "waldisk.db"))
|
|
try:
|
|
assert check.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
|
finally:
|
|
check.close()
|
|
|
|
def test_does_not_downgrade_when_disk_says_wal(self, tmp_path):
|
|
"""Refuse to downgrade an already-WAL DB even if the set-pragma path
|
|
would have raised a downgrade-eligible marker.
|
|
|
|
With the WAL-skip patch, the read-only probe short-circuits before
|
|
``PRAGMA journal_mode=WAL`` ever runs on an already-WAL connection,
|
|
so the set-pragma path is unreachable here and ``attempts`` stays 0.
|
|
Either outcome (skip-via-probe OR re-raise-on-disk-check) preserves
|
|
the property this test guards: we never silently DELETE-downgrade
|
|
a WAL-mode file. The on-disk guard remains in place as
|
|
belt-and-suspenders for any future code path that bypasses the
|
|
probe.
|
|
"""
|
|
# Prime the file in WAL mode using a normal connection
|
|
primer = sqlite3.connect(str(tmp_path / "already-wal.db"), isolation_level=None)
|
|
try:
|
|
primer.execute("PRAGMA journal_mode=WAL")
|
|
primer.execute("CREATE TABLE t (x INTEGER)")
|
|
primer.execute("INSERT INTO t VALUES (1)")
|
|
assert primer.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
|
finally:
|
|
primer.close()
|
|
|
|
# New connection whose set-WAL pragma would raise "locking protocol"
|
|
# if it were ever called. With the WAL-skip patch the probe sees
|
|
# journal_mode=wal and returns early, so set-WAL is never attempted.
|
|
conn, attempts = _open_blocking(
|
|
tmp_path / "already-wal.db",
|
|
reason="locking protocol",
|
|
isolation_level=None,
|
|
)
|
|
result = apply_wal_with_fallback(conn)
|
|
assert result == "wal", (
|
|
"must report wal mode (either skipped via probe or refused downgrade)"
|
|
)
|
|
assert attempts[0] == 0, (
|
|
"set-WAL pragma must not run when the on-disk header already says wal"
|
|
)
|
|
conn.close()
|
|
|
|
# And the file is STILL WAL on disk — nothing got rewritten
|
|
check = sqlite3.connect(str(tmp_path / "already-wal.db"))
|
|
try:
|
|
assert check.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
|
|
finally:
|
|
check.close()
|
|
|
|
def test_reraises_unrelated_operational_error(self, tmp_path):
|
|
"""Non-WAL-compat errors must NOT be silently swallowed by the fallback."""
|
|
conn, _ = _open_blocking(
|
|
tmp_path / "other.db",
|
|
reason="no such table: nope",
|
|
isolation_level=None,
|
|
)
|
|
with pytest.raises(sqlite3.OperationalError, match="no such table"):
|
|
apply_wal_with_fallback(conn)
|
|
conn.close()
|
|
|
|
def test_error_deduplicated_per_db_label(self, tmp_path, caplog):
|
|
"""Repeated calls with the same db_label log exactly ONE error.
|
|
|
|
Prevents log spam when NFS users run kanban (which opens a fresh
|
|
connection on every operation — see hermes_cli/kanban_db.py).
|
|
Regression guard: the fix for #22032 ran apply_wal_with_fallback()
|
|
on every kb.connect() call; without dedup, errors.log fills with
|
|
hundreds of identical errors per hour.
|
|
"""
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
# Three separate connections to "the same DB" via the same label
|
|
for i in range(3):
|
|
conn, _ = _open_blocking(tmp_path / f"dup-{i}.db", isolation_level=None)
|
|
mode = apply_wal_with_fallback(conn, db_label="shared.db")
|
|
assert mode == "delete"
|
|
conn.close()
|
|
|
|
# Exactly one error across all three calls
|
|
errors = [
|
|
r
|
|
for r in caplog.records
|
|
if r.levelname == "ERROR" and "shared.db" in r.getMessage()
|
|
]
|
|
assert len(errors) == 1, (
|
|
f"Expected 1 deduplicated error, got {len(errors)}: "
|
|
f"{[r.getMessage() for r in errors]}"
|
|
)
|
|
|
|
def test_falls_back_when_delete_pragma_also_fails(self, tmp_path, caplog):
|
|
"""WAL-incompat FS that ALSO rejects DELETE — must not crash callers (#30816).
|
|
|
|
``PRAGMA journal_mode=WAL`` raises a recognized WAL-incompat marker
|
|
(``locking protocol``), so the DELETE fallback engages — but DELETE
|
|
*also* raises (``disk I/O error``, observed on APFS external SSDs
|
|
under heavy contention). The connection's default journal_mode is
|
|
already DELETE, so it is still usable; propagating would crash
|
|
SessionDB / kanban_db / ResponseStore init. Returns ``"delete"``
|
|
and logs one WARNING per db_label.
|
|
|
|
Fault injection only: the APFS trigger itself is not reproducible on
|
|
Linux — this drives the same code path with a failing DELETE pragma.
|
|
"""
|
|
delete_attempts = [0]
|
|
|
|
class _DeletePragmaFailsConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
lowered = sql.lower().replace(" ", "")
|
|
if "journal_mode=wal" in lowered:
|
|
raise sqlite3.OperationalError("locking protocol")
|
|
if "journal_mode=delete" in lowered:
|
|
delete_attempts[0] += 1
|
|
raise sqlite3.OperationalError("disk I/O error")
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "apfs.db"),
|
|
factory=_DeletePragmaFailsConnection,
|
|
isolation_level=None,
|
|
)
|
|
with caplog.at_level("WARNING", logger="hermes_state"):
|
|
mode = apply_wal_with_fallback(conn, db_label="apfs-test.db")
|
|
|
|
assert mode == "delete"
|
|
assert delete_attempts[0] == 1
|
|
|
|
msgs = [r.getMessage() for r in caplog.records if r.levelname in ("WARNING", "ERROR")]
|
|
assert any("apfs-test.db" in m and "WAL" in m for m in msgs)
|
|
delete_warnings = [
|
|
m for m in msgs if "apfs-test.db" in m and "both WAL and DELETE journal_mode failed" in m
|
|
]
|
|
assert len(delete_warnings) == 1
|
|
assert "disk I/O error" in delete_warnings[0]
|
|
|
|
# Connection is still usable for non-journal_mode SQL
|
|
conn.execute("CREATE TABLE t (x INTEGER)")
|
|
conn.execute("INSERT INTO t VALUES (1)")
|
|
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
|
|
conn.close()
|
|
|
|
def test_both_pragmas_fail_but_readback_reports_actual_mode(self, tmp_path, caplog):
|
|
"""When both PRAGMA writes fail, the return value is read back from the
|
|
connection — not a hardcoded ``"delete"`` guess.
|
|
|
|
The connection here already runs ``journal_mode=MEMORY`` (set before
|
|
the failure injection); both WAL and DELETE writes then fail, so the
|
|
function must report the actual ``"memory"`` mode.
|
|
"""
|
|
|
|
class _WritesFailReadsSucceedConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
lowered = sql.lower().replace(" ", "")
|
|
if "journal_mode=wal" in lowered:
|
|
raise sqlite3.OperationalError("locking protocol")
|
|
if "journal_mode=delete" in lowered:
|
|
raise sqlite3.OperationalError("disk I/O error")
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "readback.db"),
|
|
factory=_WritesFailReadsSucceedConnection,
|
|
isolation_level=None,
|
|
)
|
|
conn.execute("PRAGMA journal_mode=MEMORY")
|
|
with caplog.at_level("WARNING", logger="hermes_state"):
|
|
mode = apply_wal_with_fallback(conn, db_label="readback.db")
|
|
|
|
assert mode == "memory"
|
|
msgs = [r.getMessage() for r in caplog.records if r.levelname in ("WARNING", "ERROR")]
|
|
assert any(
|
|
"readback.db" in m and "both WAL and DELETE journal_mode failed" in m
|
|
for m in msgs
|
|
)
|
|
conn.close()
|
|
|
|
def test_delete_fallback_failure_warning_deduplicated_per_db_label(self, tmp_path, caplog):
|
|
"""Repeated both-fail calls with the same db_label log exactly ONE WARNING.
|
|
|
|
kanban_db.connect() runs on every kanban operation; without dedup,
|
|
APFS-external-SSD users would see hundreds of identical warnings.
|
|
"""
|
|
|
|
class _BothPragmasFailConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
lowered = sql.lower().replace(" ", "")
|
|
if "journal_mode=wal" in lowered:
|
|
raise sqlite3.OperationalError("locking protocol")
|
|
if "journal_mode=delete" in lowered:
|
|
raise sqlite3.OperationalError("disk I/O error")
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
with caplog.at_level("WARNING", logger="hermes_state"):
|
|
for i in range(3):
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / f"apfs-dup-{i}.db"),
|
|
factory=_BothPragmasFailConnection,
|
|
isolation_level=None,
|
|
)
|
|
assert apply_wal_with_fallback(conn, db_label="apfs-shared.db") == "delete"
|
|
conn.close()
|
|
|
|
delete_warnings = [
|
|
r for r in caplog.records
|
|
if r.levelname == "WARNING"
|
|
and "apfs-shared.db" in r.getMessage()
|
|
and "both WAL and DELETE journal_mode failed" in r.getMessage()
|
|
]
|
|
assert len(delete_warnings) == 1, (
|
|
f"Expected 1 deduplicated DELETE-failed warning, got {len(delete_warnings)}"
|
|
)
|
|
|
|
def test_error_fires_independently_per_db_label(self, tmp_path, caplog):
|
|
"""Different db_labels each get their own one error (not globally dedup'd)."""
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
conn1, _ = _open_blocking(tmp_path / "a.db", isolation_level=None)
|
|
apply_wal_with_fallback(conn1, db_label="state.db")
|
|
conn1.close()
|
|
|
|
conn2, _ = _open_blocking(tmp_path / "b.db", isolation_level=None)
|
|
apply_wal_with_fallback(conn2, db_label="kanban.db")
|
|
conn2.close()
|
|
|
|
errors = [r for r in caplog.records if r.levelname == "ERROR"]
|
|
labels_logged = {
|
|
lbl
|
|
for r in errors
|
|
for lbl in ("state.db", "kanban.db")
|
|
if lbl in r.getMessage()
|
|
}
|
|
assert labels_logged == {"state.db", "kanban.db"}, (
|
|
f"Each db_label should log once; got {labels_logged}"
|
|
)
|
|
|
|
|
|
class TestRequireWal:
|
|
"""``require_wal=True`` turns either WAL-refusal shape into a hard
|
|
``WalUnsupportedError`` for callers that mandate WAL concurrency, instead
|
|
of silently degrading to DELETE. The default callers (SessionDB /
|
|
kanban_db.connect) keep ``require_wal=False`` so NFS-homed installs work.
|
|
"""
|
|
|
|
def test_happy_path_unaffected_by_require_wal(self, tmp_path):
|
|
"""On a WAL-capable FS, require_wal=True still simply returns 'wal'."""
|
|
conn = sqlite3.connect(str(tmp_path / "ok.db"), isolation_level=None)
|
|
try:
|
|
assert apply_wal_with_fallback(conn, require_wal=True) == "wal"
|
|
finally:
|
|
conn.close()
|
|
|
|
def test_raises_on_raising_marker(self, tmp_path):
|
|
"""NFS-style raising refusal + require_wal=True → WalUnsupportedError."""
|
|
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
|
|
try:
|
|
with pytest.raises(WalUnsupportedError, match="locking protocol"):
|
|
apply_wal_with_fallback(conn, db_label="state.db", require_wal=True)
|
|
finally:
|
|
conn.close()
|
|
|
|
def test_raises_on_silent_refusal(self, tmp_path):
|
|
"""macOS-NFS silent no-op + require_wal=True → WalUnsupportedError,
|
|
not a false 'delete' return."""
|
|
factory = _make_silent_noop_factory("delete")
|
|
conn = sqlite3.connect(
|
|
str(tmp_path / "macnfs.db"), factory=factory, isolation_level=None
|
|
)
|
|
try:
|
|
with pytest.raises(WalUnsupportedError, match="refused without raising"):
|
|
apply_wal_with_fallback(conn, db_label="kanban.db", require_wal=True)
|
|
finally:
|
|
conn.close()
|
|
|
|
def test_error_is_operationalerror_subclass(self, tmp_path):
|
|
"""WalUnsupportedError must stay catchable as sqlite3.OperationalError
|
|
so existing DB-init handlers (e.g. SessionDB.__init__) still catch it."""
|
|
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
|
|
try:
|
|
with pytest.raises(sqlite3.OperationalError):
|
|
apply_wal_with_fallback(conn, require_wal=True)
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
class TestGetLastInitError:
|
|
|
|
|
|
def test_captures_cause_on_failed_init(self, tmp_path):
|
|
"""When SessionDB() raises, the cause is preserved for slash commands.
|
|
|
|
Simulates a filesystem failure unrelated to the WAL fallback path —
|
|
e.g. ``foreign_keys=ON`` rejected by a read-only or seriously
|
|
damaged DB. (Both-pragmas-fail no longer raises since #30816: the
|
|
DELETE fallback is guarded and returns the mode in effect.) The
|
|
exception bubbles out of ``SessionDB.__init__`` and the cause is
|
|
captured for /resume to surface.
|
|
"""
|
|
target = tmp_path / "broken.db"
|
|
real_connect = sqlite3.connect
|
|
|
|
class _ForeignKeysFailConnection(sqlite3.Connection):
|
|
def execute(self, sql, *args, **kwargs): # type: ignore[override]
|
|
if "foreign_keys=on" in sql.lower().replace(" ", ""):
|
|
raise sqlite3.OperationalError(
|
|
"foreign_keys=ON rejected: read-only filesystem"
|
|
)
|
|
return super().execute(sql, *args, **kwargs)
|
|
|
|
def gated_connect(*args, **kwargs):
|
|
# connect_tracked passes a tracking-augmented factory; drop it and
|
|
# substitute the double, which connect_tracked will re-augment.
|
|
kwargs.pop("factory", None)
|
|
return real_connect(str(target), factory=_ForeignKeysFailConnection, **kwargs)
|
|
|
|
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
|
|
with pytest.raises(sqlite3.OperationalError):
|
|
SessionDB(db_path=target)
|
|
|
|
cause = get_last_init_error()
|
|
assert cause is not None
|
|
assert "OperationalError" in cause
|
|
assert "read-only filesystem" in cause
|
|
|
|
|
|
|
|
|
|
class TestSessionDbUsesWalFallback:
|
|
def test_sessiondb_works_when_wal_unavailable(self, tmp_path):
|
|
"""E2E: SessionDB initializes and performs a write on a WAL-blocked FS."""
|
|
target = tmp_path / "nfs_style.db"
|
|
|
|
real_connect = sqlite3.connect
|
|
attempts = [0]
|
|
factory = _make_blocking_factory("locking protocol", attempts)
|
|
|
|
def gated_connect(*args, **kwargs):
|
|
# connect_tracked passes a tracking-augmented factory; drop it and
|
|
# substitute the double, which connect_tracked re-applies to the
|
|
# returned instance.
|
|
kwargs.pop("factory", None)
|
|
return real_connect(str(target), factory=factory, **kwargs)
|
|
|
|
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
|
|
db = SessionDB(db_path=target)
|
|
|
|
try:
|
|
# WAL was attempted and rejected — fallback kicked in
|
|
assert attempts[0] >= 1, (
|
|
"WAL pragma was never executed — check the patch target"
|
|
)
|
|
# SessionDB is usable end-to-end: create a session, read it back
|
|
db.create_session(session_id="s1", source="cli", model="test")
|
|
sess = db.get_session("s1")
|
|
assert sess is not None
|
|
assert sess["source"] == "cli"
|
|
# No init error was recorded since init succeeded via the fallback
|
|
assert get_last_init_error() is None
|
|
finally:
|
|
db.close()
|
|
|
|
def test_sessiondb_works_when_wal_is_silently_refused(self, tmp_path, caplog):
|
|
"""E2E: SessionDB stays usable when WAL silently no-ops to DELETE."""
|
|
target = tmp_path / "macnfs_style.db"
|
|
|
|
real_connect = sqlite3.connect
|
|
factory = _make_silent_noop_factory("delete")
|
|
|
|
def gated_connect(*args, **kwargs):
|
|
# connect_tracked passes a tracking-augmented factory; drop it and
|
|
# substitute the double, which connect_tracked re-applies to the
|
|
# returned instance.
|
|
kwargs.pop("factory", None)
|
|
return real_connect(str(target), factory=factory, **kwargs)
|
|
|
|
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
|
|
with caplog.at_level("ERROR", logger="hermes_state"):
|
|
db = SessionDB(db_path=target)
|
|
|
|
try:
|
|
assert db._conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "delete"
|
|
db.create_session(session_id="s2", source="cli", model="test")
|
|
sess = db.get_session("s2")
|
|
assert sess is not None
|
|
assert sess["source"] == "cli"
|
|
assert get_last_init_error() is None
|
|
finally:
|
|
db.close()
|
|
|
|
errors = [
|
|
r
|
|
for r in caplog.records
|
|
if r.levelname == "ERROR" and "state.db" in r.getMessage()
|
|
]
|
|
assert len(errors) == 1, (
|
|
"silent WAL refusal should emit exactly one state.db error"
|
|
)
|