hermes-agent/tests/hermes_state/test_hermes_state_wal_fallback.py
ethernet 890bbbda1f Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts
#	apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts
#	apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts
#	apps/desktop/e2e/bot-mode-roster-localized.spec.ts
#	apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts
#	apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts
#	apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts
#	apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts
#	apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts
#	apps/desktop/e2e/bot-roster-user-sections.spec.ts
#	apps/desktop/e2e/bot-routines-pane-narrow.spec.ts
#	apps/desktop/e2e/bot-row-open-recent-session.spec.ts
#	apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts
#	apps/desktop/e2e/group-composer-auto-grow.spec.ts
#	apps/desktop/e2e/group-create-gate-remote-roster.spec.ts
#	apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts
#	apps/desktop/e2e/hosted-room-backend-continuity.spec.ts
#	apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts
#	apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts
#	apps/desktop/e2e/worktree-branch-status.spec.ts
#	apps/desktop/electron/backend-probes.test.ts
#	apps/desktop/electron/connection-apply.test.ts
#	apps/desktop/electron/desktop-electron-pin.test.ts
#	apps/desktop/electron/desktop-uninstall.test.ts
#	apps/desktop/electron/gateway-file-download-transport.test.ts
#	apps/desktop/electron/gateway-stop-before-update.test.ts
#	apps/desktop/electron/github-api-auth.test.ts
#	apps/desktop/electron/registry-primary-profile-scope.test.ts
#	apps/desktop/electron/update-api-check.test.ts
#	apps/desktop/electron/update-handoff-marker.test.ts
#	apps/desktop/electron/venv-blocker-scan.test.ts
#	apps/desktop/scripts/after-extract.test.mjs
#	apps/desktop/scripts/local-pack-publish.test.mjs
#	apps/desktop/scripts/tasks-scroll.test.mjs
#	apps/desktop/src/app/settings/model-settings.test.tsx
#	apps/desktop/src/app/updates-overlay.blockers.test.tsx
#	apps/desktop/src/components/desktop-install-overlay.test.tsx
#	apps/desktop/src/lib/update-copy.test.ts
#	scripts/ci/check_os_marker_fakes.py
#	tests-js/desktop-mac-usage-descriptions.test.ts
#	tests-js/node-engine-alignment.test.ts
#	tests/agent/lsp/test_install_and_lint_fixes.py
#	tests/agent/test_command_token_source.py
#	tests/agent/test_compression_boundary_hook.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/agent/test_custom_provider_ca_probes.py
#	tests/agent/test_endpoint_blackhole.py
#	tests/agent/test_estimator_parity.py
#	tests/agent/test_in_place_compaction.py
#	tests/agent/test_moa_loop_mode.py
#	tests/agent/test_model_metadata.py
#	tests/agent/test_skill_session_platform_gate.py
#	tests/agent/test_skill_utils.py
#	tests/agent/test_ssl_ca_guard.py
#	tests/computer_use/test_doctor.py
#	tests/cron/test_codex_execution_paths.py
#	tests/cron/test_cron_bot_chat_delivery.py
#	tests/cron/test_cron_script.py
#	tests/cron/test_media_delivery_parity.py
#	tests/cron/test_misfire_catchup.py
#	tests/cron/test_parallel_pool.py
#	tests/cron/test_recurring_eagain_redispatch.py
#	tests/gateway/test_choice_picker.py
#	tests/gateway/test_control_socket_windows_live.py
#	tests/gateway/test_dingtalk.py
#	tests/gateway/test_feishu.py
#	tests/gateway/test_feishu_onboard.py
#	tests/gateway/test_gateway_shutdown.py
#	tests/gateway/test_matrix.py
#	tests/gateway/test_model_command_custom_providers.py
#	tests/gateway/test_reasoning_command.py
#	tests/gateway/test_runtime_footer.py
#	tests/gateway/test_session.py
#	tests/gateway/test_session_hygiene.py
#	tests/gateway/test_status.py
#	tests/gateway/test_teams.py
#	tests/gateway/test_turn_lease.py
#	tests/gateway/test_whatsapp_connect.py
#	tests/hermes_cli/test_approvals_command.py
#	tests/hermes_cli/test_auth_store_lock_concurrent.py
#	tests/hermes_cli/test_backup.py
#	tests/hermes_cli/test_banner_git_state.py
#	tests/hermes_cli/test_certifi_repair.py
#	tests/hermes_cli/test_cmd_update.py
#	tests/hermes_cli/test_compat_manifest_targets.py
#	tests/hermes_cli/test_computer_use_cli.py
#	tests/hermes_cli/test_cpr_local_leak.py
#	tests/hermes_cli/test_dashboard_auth_gate.py
#	tests/hermes_cli/test_dashboard_procs_kill_grace.py
#	tests/hermes_cli/test_desktop_lifecycle_windows_live.py
#	tests/hermes_cli/test_doctor.py
#	tests/hermes_cli/test_doctor_command_install.py
#	tests/hermes_cli/test_fleet_config_migration_windows_live.py
#	tests/hermes_cli/test_gateway.py
#	tests/hermes_cli/test_gateway_platform_gating.py
#	tests/hermes_cli/test_gateway_restart_loop.py
#	tests/hermes_cli/test_gateway_task_probe.py
#	tests/hermes_cli/test_gateway_wsl.py
#	tests/hermes_cli/test_gui_command.py
#	tests/hermes_cli/test_install_cua_driver.py
#	tests/hermes_cli/test_kanban_db.py
#	tests/hermes_cli/test_lazy_command_exports.py
#	tests/hermes_cli/test_lazy_refresh_venv_repair.py
#	tests/hermes_cli/test_linux_desktop_entry.py
#	tests/hermes_cli/test_local_runtime.py
#	tests/hermes_cli/test_local_runtime_updates.py
#	tests/hermes_cli/test_managed_uv.py
#	tests/hermes_cli/test_mcp_reload_confirm_gate.py
#	tests/hermes_cli/test_nous_subscription.py
#	tests/hermes_cli/test_npm_engine.py
#	tests/hermes_cli/test_personality_none.py
#	tests/hermes_cli/test_pet_toggle.py
#	tests/hermes_cli/test_plan_reconciliation_windows_live.py
#	tests/hermes_cli/test_plugin_event_bus.py
#	tests/hermes_cli/test_plugin_manifest_v2.py
#	tests/hermes_cli/test_plugin_packs.py
#	tests/hermes_cli/test_plugins_cmd.py
#	tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py
#	tests/hermes_cli/test_process_identity.py
#	tests/hermes_cli/test_profiles.py
#	tests/hermes_cli/test_profiles_sidebar_cache.py
#	tests/hermes_cli/test_pty_bridge.py
#	tests/hermes_cli/test_resolve_turn_limit.py
#	tests/hermes_cli/test_serve_runtime_inventory.py
#	tests/hermes_cli/test_session_vacuum_config.py
#	tests/hermes_cli/test_set_config_value.py
#	tests/hermes_cli/test_signal_handler_kanban_worker.py
#	tests/hermes_cli/test_slash_confirm_windows.py
#	tests/hermes_cli/test_stale_pid_guard.py
#	tests/hermes_cli/test_startup_fast_guards.py
#	tests/hermes_cli/test_status.py
#	tests/hermes_cli/test_telegram_managed_bot.py
#	tests/hermes_cli/test_tools_config.py
#	tests/hermes_cli/test_update_apply_shallow_count.py
#	tests/hermes_cli/test_update_autostash.py
#	tests/hermes_cli/test_update_concurrent_quarantine.py
#	tests/hermes_cli/test_update_fetch_failure_classifier.py
#	tests/hermes_cli/test_update_fleet_probe_resume_token.py
#	tests/hermes_cli/test_update_handoff_backend_reap.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_update_head_moved_gate.py
#	tests/hermes_cli/test_update_host_obligation.py
#	tests/hermes_cli/test_update_import_guard.py
#	tests/hermes_cli/test_update_interrupted_recovery.py
#	tests/hermes_cli/test_update_inventory.py
#	tests/hermes_cli/test_update_launchd_unloaded_gateway.py
#	tests/hermes_cli/test_update_missing_configured_deps.py
#	tests/hermes_cli/test_update_modified_notice.py
#	tests/hermes_cli/test_update_multiplex_migration_hook.py
#	tests/hermes_cli/test_update_no_gateway_restart.py
#	tests/hermes_cli/test_update_orphan_backend_reap.py
#	tests/hermes_cli/test_update_parked_branch_guard.py
#	tests/hermes_cli/test_update_post_pull_syntax_guard.py
#	tests/hermes_cli/test_update_receipt.py
#	tests/hermes_cli/test_update_self_lock.py
#	tests/hermes_cli/test_update_shim_fail_closed.py
#	tests/hermes_cli/test_update_shim_self_lock.py
#	tests/hermes_cli/test_update_sqlite_remediation.py
#	tests/hermes_cli/test_update_stale_dashboard.py
#	tests/hermes_cli/test_update_stale_virtualenv.py
#	tests/hermes_cli/test_update_venv_health.py
#	tests/hermes_cli/test_update_venv_ownership_preflight.py
#	tests/hermes_cli/test_update_wedged_gateway.py
#	tests/hermes_cli/test_update_yes_flag.py
#	tests/hermes_cli/test_update_zip_two_phase.py
#	tests/hermes_cli/test_urllib_security.py
#	tests/hermes_cli/test_ux_messages_auth_config.py
#	tests/hermes_cli/test_ux_messages_startup.py
#	tests/hermes_cli/test_venv_holder_classifier.py
#	tests/hermes_cli/test_verify_console_scripts.py
#	tests/hermes_cli/test_verify_core_dependencies.py
#	tests/hermes_cli/test_web_server.py
#	tests/hermes_cli/test_web_server_console_ws.py
#	tests/hermes_cli/test_web_server_ws_ping.py
#	tests/hermes_cli/test_web_ui_build.py
#	tests/hermes_state/test_fts_rebuild_admission.py
#	tests/hermes_state/test_hermes_state.py
#	tests/plugins/memory/test_memory_lazy_install.py
#	tests/plugins/test_google_meet_plugin.py
#	tests/plugins/test_langfuse_plugin.py
#	tests/plugins/test_security_guidance_plugin.py
#	tests/plugins/test_transform_llm_output_hook.py
#	tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_contributor_map.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/skills/test_competitor_news_monitor_skill.py
#	tests/skills/test_document_to_action_items_skill.py
#	tests/skills/test_google_workspace_setup.py
#	tests/skills/test_google_workspace_setup_deps.py
#	tests/skills/test_grounded_citations_skill.py
#	tests/skills/test_ip_as_logo_skill.py
#	tests/skills/test_live_dashboard_skill.py
#	tests/skills/test_mcp_oauth_remote_gateway_skill.py
#	tests/skills/test_office_document_skills.py
#	tests/skills/test_openclaw_migration.py
#	tests/skills/test_product_price_monitor_skill.py
#	tests/skills/test_scrollcraft_skill.py
#	tests/skills/test_setup_wizard_generator_skill.py
#	tests/skills/test_weekly_review_planning_skill.py
#	tests/test_engines_satisfiable.py
#	tests/test_fast_safe_load.py
#	tests/test_hermes_bootstrap.py
#	tests/test_hermes_constants.py
#	tests/test_hermes_logging.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_model_tools_async_bridge.py
#	tests/test_packaging_build_guard.py
#	tests/test_packaging_metadata.py
#	tests/test_yaml_indent_consistency.py
#	tests/tools/test_approval_timeout_overflow.py
#	tests/tools/test_base_environment.py
#	tests/tools/test_bot_mode_dm.py
#	tests/tools/test_browser_chromium_check.py
#	tests/tools/test_browser_hardening.py
#	tests/tools/test_browser_homebrew_paths.py
#	tests/tools/test_browser_npx_warmup.py
#	tests/tools/test_browser_orphan_reaper.py
#	tests/tools/test_browser_real_profile.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_clipboard.py
#	tests/tools/test_code_execution.py
#	tests/tools/test_code_execution_modes.py
#	tests/tools/test_code_execution_windows_env.py
#	tests/tools/test_computer_use.py
#	tests/tools/test_delegate_liveness_timeout.py
#	tests/tools/test_execute_code_approval_cluster.py
#	tests/tools/test_execution_flag_detection.py
#	tests/tools/test_fal_common.py
#	tests/tools/test_file_operations.py
#	tests/tools/test_file_tools.py
#	tests/tools/test_file_tools_cwd_resolution.py
#	tests/tools/test_file_tools_live.py
#	tests/tools/test_lazy_deps.py
#	tests/tools/test_lazy_deps_durable_target.py
#	tests/tools/test_lazy_deps_managed.py
#	tests/tools/test_local_env_blocklist.py
#	tests/tools/test_local_tempdir.py
#	tests/tools/test_macos_protected_search.py
#	tests/tools/test_mcp_npx_cached_bin.py
#	tests/tools/test_oneshot_completion_linger.py
#	tests/tools/test_process_registry.py
#	tests/tools/test_read_file_schema_gating.py
#	tests/tools/test_skill_improvements.py
#	tests/tools/test_skills_sync.py
#	tests/tools/test_termux_api_detection.py
#	tests/tools/test_tirith_security.py
#	tests/tools/test_transcription_tools.py
#	tests/tools/test_tts_streaming.py
#	tests/tools/test_wake_word.py
#	tests/tui_gateway/test_compute_host_borrowed_lease.py
#	tests/tui_gateway/test_compute_host_turn_protocol.py
#	tests/tui_gateway/test_isolated_orphan_activity.py
#	tests/tui_gateway/test_protocol.py
#	tests/tui_gateway/test_slash_worker_profile_home.py
#	tests/tui_gateway/test_subprocess_encoding.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	ui-tui/src/__tests__/terminalParity.test.ts
#	ui-tui/src/__tests__/termuxComposerLayout.test.ts
#	ui-tui/src/__tests__/textInputFastEcho.test.ts
2026-09-23 07:02:44 -04:00

659 lines
29 KiB
Python

"""Tests for the WAL→DELETE journal-mode fallback on NFS / SMB / FUSE.
When ``PRAGMA journal_mode=WAL`` raises ``OperationalError("locking protocol")``
(SQLITE_PROTOCOL — typical on NFS/SMB), Hermes must fall back to
``journal_mode=DELETE`` so ``state.db`` / ``kanban.db`` remain usable.
Without this fallback, users on NFS-mounted ``HERMES_HOME`` silently lose
``/resume``, ``/title``, ``/history``, ``/branch``, session search, and the
kanban dispatcher — because ``SessionDB()`` init propagates the error and
every caller swallows it, leaving ``_session_db = None``.
See: https://www.sqlite.org/wal.html — "WAL does not work over a network
filesystem".
"""
import sqlite3
from unittest.mock import patch
import pytest
import hermes_state
import hermes_state_wal
from hermes_state import SessionDB, get_last_init_error
from hermes_state_wal import WalUnsupportedError, apply_wal_with_fallback
# ``sqlite3.Connection.execute`` is a C-level slot and can't be monkeypatched
# directly (``'sqlite3.Connection' object attribute 'execute' is read-only``).
# A factory-built subclass lets us intercept journal_mode=WAL per-test with
# its own mutable counter, avoiding a parallel-run class-state race.
def _make_blocking_factory(reason: str, attempt_counter: list):
"""Return a sqlite3.Connection subclass that raises on PRAGMA journal_mode=WAL."""
class _WalBlockingConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
if "journal_mode=wal" in sql.lower().replace(" ", ""):
attempt_counter[0] += 1
raise sqlite3.OperationalError(reason)
return super().execute(sql, *args, **kwargs)
return _WalBlockingConnection
def _open_blocking(path, reason="locking protocol", **kwargs):
"""Open a connection whose WAL pragma raises ``reason``.
Returns ``(conn, attempt_counter_list)`` so callers can assert how many
times WAL was attempted.
"""
attempts = [0]
factory = _make_blocking_factory(reason, attempts)
return sqlite3.connect(str(path), factory=factory, **kwargs), attempts
def _make_silent_noop_factory(returned_mode: str = "delete"):
"""Return a ``sqlite3.Connection`` subclass whose ``PRAGMA journal_mode=WAL``
silently NO-OPs: it returns the still-effective journal mode (e.g.
``delete``) WITHOUT raising — the way macOS NFS / SMB / the AgentFS NFS
overlay behave. NFS that *raises* SQLITE_PROTOCOL is covered by
:func:`_make_blocking_factory`; this is the other, quieter failure shape.
"""
class _WalSilentNoOpConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
if "journal_mode=wal" in sql.lower().replace(" ", ""):
# Refuse the WAL switch but DON'T raise; report the mode that is
# actually still in force, exactly as the NFS client does.
return super().execute(
f"PRAGMA journal_mode={returned_mode}", *args, **kwargs
)
return super().execute(sql, *args, **kwargs)
return _WalSilentNoOpConnection
@pytest.fixture(autouse=True)
def _reset_last_init_error():
"""Reset the module-global last-error before and after each test."""
hermes_state._set_last_init_error(None)
yield
hermes_state._set_last_init_error(None)
@pytest.fixture(autouse=True)
def _reset_wal_fallback_warned_paths():
"""Reset the WAL-fallback warned-paths set so dedup doesn't leak between tests."""
hermes_state_wal._wal_fallback_warned_paths.clear()
hermes_state_wal._wal_delete_fallback_failed_paths.clear()
yield
hermes_state_wal._wal_fallback_warned_paths.clear()
hermes_state_wal._wal_delete_fallback_failed_paths.clear()
@pytest.fixture(autouse=True)
def _assume_fixed_sqlite(monkeypatch):
"""NFS-fallback tests assume a SQLite build without the WAL-reset bug."""
monkeypatch.setattr(
hermes_state_wal, "is_sqlite_wal_reset_vulnerable", lambda version_info=None: False
)
hermes_state_wal._wal_reset_bug_warned_paths.clear()
yield
hermes_state_wal._wal_reset_bug_warned_paths.clear()
class TestApplyWalWithFallback:
def test_succeeds_on_local_fs(self, tmp_path):
"""Happy path: WAL works on a normal filesystem."""
conn = sqlite3.connect(str(tmp_path / "ok.db"), isolation_level=None)
mode = apply_wal_with_fallback(conn)
assert mode == "wal"
cur = conn.execute("PRAGMA journal_mode")
assert cur.fetchone()[0].lower() == "wal"
conn.close()
def test_falls_back_to_delete_on_locking_protocol(self, tmp_path, caplog):
"""NFS-style ``locking protocol`` error → DELETE mode + one ERROR."""
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
with caplog.at_level("ERROR", logger="hermes_state"):
mode = apply_wal_with_fallback(conn, db_label="test.db")
assert mode == "delete"
errors = [r for r in caplog.records if r.levelname == "ERROR"]
assert len(errors) == 1
msg = errors[0].getMessage()
assert "test.db" in msg
assert "journal_mode=DELETE" in msg
assert "locking protocol" in msg
# Post-fallback the DB is still usable for real writes
conn.execute("CREATE TABLE t (x INTEGER)")
conn.execute("INSERT INTO t VALUES (1)")
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
conn.close()
def test_falls_back_on_not_authorized(self, tmp_path):
"""Some FUSE mounts block WAL pragma outright ('not authorized')."""
conn, _ = _open_blocking(
tmp_path / "fuse.db", reason="not authorized", isolation_level=None
)
mode = apply_wal_with_fallback(conn)
assert mode == "delete"
conn.close()
def test_falls_back_when_wal_silently_refused(self, tmp_path, caplog):
"""macOS NFS / SMB / AgentFS-NFS can REFUSE the WAL switch WITHOUT
raising: ``PRAGMA journal_mode=WAL`` returns the still-effective mode
('delete') and no exception. The marker-exception path never fires, so
the function must detect the silent no-op from the PRAGMA's RETURN
value — otherwise it returns a false 'wal' and skips the WARNING.
Reproduced on a real AgentFS NFS overlay, where
``PRAGMA journal_mode=WAL`` returned ('delete',) with no
OperationalError.
"""
factory = _make_silent_noop_factory("delete")
conn = sqlite3.connect(
str(tmp_path / "macnfs.db"), factory=factory, isolation_level=None
)
with caplog.at_level("ERROR", logger="hermes_state"):
mode = apply_wal_with_fallback(conn, db_label="kanban.db")
assert mode == "delete", "must report the true mode, not a false 'wal'"
assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "delete"
errors = [r for r in caplog.records if r.levelname == "ERROR"]
assert len(errors) == 1, "silent no-op must still emit exactly one ERROR"
assert "kanban.db" in errors[0].getMessage()
conn.close()
def test_transient_disk_io_error_recovers_to_wal(self, tmp_path):
"""Transient EIO from ``PRAGMA journal_mode=WAL`` must NOT silently
downgrade to DELETE.
Regression for "Bug D": treating transient EIO as a permanent
WAL-incompat marker produced the mixed-journal-mode-across-processes
corruption pattern (process A downgrades to DELETE, sibling
processes successfully set WAL, SQLite corrupts the file because
the two locking protocols are documented as incompatible). EIO is
often transient (page-cache pressure, lock contention, brief
storage hiccups); the fallback retries the pragma, and a transient
EIO clears — returning "wal", never walking the DB into a
permanently downgraded state.
"""
attempts = [0]
class _TransientEIOConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
if "journal_mode=wal" in sql.lower().replace(" ", ""):
attempts[0] += 1
if attempts[0] == 1:
raise sqlite3.OperationalError("disk I/O error")
return super().execute(sql, *args, **kwargs)
conn = sqlite3.connect(
str(tmp_path / "flaky.db"),
factory=_TransientEIOConnection,
isolation_level=None,
)
assert apply_wal_with_fallback(conn) == "wal"
assert attempts[0] == 2, "the retry must have re-attempted the pragma"
assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
conn.close()
def test_persistent_disk_io_error_falls_back_to_delete(self, tmp_path, caplog):
"""Deterministic EIO (ZFS / APFS-CoW SHM corruption — #55305, #71498)
keeps failing across retries → guarded fallback to DELETE with one
ERROR, instead of wedging the session DB entirely.
"""
conn, attempts = _open_blocking(
tmp_path / "zfs.db", reason="disk I/O error", isolation_level=None
)
with caplog.at_level("ERROR", logger="hermes_state"):
mode = apply_wal_with_fallback(conn, db_label="zfs.db")
assert mode == "delete"
assert attempts[0] >= 3, "must retry before concluding EIO is persistent"
errors = [
r
for r in caplog.records
if r.levelname == "ERROR" and "zfs.db" in r.getMessage()
]
assert len(errors) == 1
# Post-fallback the DB is still usable for real writes
conn.execute("CREATE TABLE t (x INTEGER)")
conn.execute("INSERT INTO t VALUES (1)")
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
conn.close()
def test_persistent_disk_io_error_never_downgrades_wal_disk(self, tmp_path):
"""Persistent EIO on a DB whose on-disk header already says WAL must
re-raise (Bug D guard) — never flip a WAL file to DELETE under
possible concurrent openers.
"""
primer = sqlite3.connect(str(tmp_path / "waldisk.db"), isolation_level=None)
try:
primer.execute("PRAGMA journal_mode=WAL")
primer.execute("CREATE TABLE t (x INTEGER)")
finally:
primer.close()
class _EIOWalOnlyConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
normalized = sql.lower().replace(" ", "")
if "journal_mode=wal" in normalized:
raise sqlite3.OperationalError("disk I/O error")
return super().execute(sql, *args, **kwargs)
conn = sqlite3.connect(
str(tmp_path / "waldisk.db"),
factory=_EIOWalOnlyConnection,
isolation_level=None,
)
# The read-only probe sees WAL and returns early on this shape; force
# the set-pragma path by making the probe report a non-WAL mode.
# (On-disk header still says WAL, which is what the guard checks.)
try:
result = apply_wal_with_fallback(conn)
# Probe short-circuit: acceptable, DB stays WAL.
assert result == "wal"
except sqlite3.OperationalError:
pass # re-raise path: equally acceptable — no downgrade happened
finally:
conn.close()
check = sqlite3.connect(str(tmp_path / "waldisk.db"))
try:
assert check.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
finally:
check.close()
def test_does_not_downgrade_when_disk_says_wal(self, tmp_path):
"""Refuse to downgrade an already-WAL DB even if the set-pragma path
would have raised a downgrade-eligible marker.
With the WAL-skip patch, the read-only probe short-circuits before
``PRAGMA journal_mode=WAL`` ever runs on an already-WAL connection,
so the set-pragma path is unreachable here and ``attempts`` stays 0.
Either outcome (skip-via-probe OR re-raise-on-disk-check) preserves
the property this test guards: we never silently DELETE-downgrade
a WAL-mode file. The on-disk guard remains in place as
belt-and-suspenders for any future code path that bypasses the
probe.
"""
# Prime the file in WAL mode using a normal connection
primer = sqlite3.connect(str(tmp_path / "already-wal.db"), isolation_level=None)
try:
primer.execute("PRAGMA journal_mode=WAL")
primer.execute("CREATE TABLE t (x INTEGER)")
primer.execute("INSERT INTO t VALUES (1)")
assert primer.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
finally:
primer.close()
# New connection whose set-WAL pragma would raise "locking protocol"
# if it were ever called. With the WAL-skip patch the probe sees
# journal_mode=wal and returns early, so set-WAL is never attempted.
conn, attempts = _open_blocking(
tmp_path / "already-wal.db",
reason="locking protocol",
isolation_level=None,
)
result = apply_wal_with_fallback(conn)
assert result == "wal", (
"must report wal mode (either skipped via probe or refused downgrade)"
)
assert attempts[0] == 0, (
"set-WAL pragma must not run when the on-disk header already says wal"
)
conn.close()
# And the file is STILL WAL on disk — nothing got rewritten
check = sqlite3.connect(str(tmp_path / "already-wal.db"))
try:
assert check.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal"
finally:
check.close()
def test_reraises_unrelated_operational_error(self, tmp_path):
"""Non-WAL-compat errors must NOT be silently swallowed by the fallback."""
conn, _ = _open_blocking(
tmp_path / "other.db",
reason="no such table: nope",
isolation_level=None,
)
with pytest.raises(sqlite3.OperationalError, match="no such table"):
apply_wal_with_fallback(conn)
conn.close()
def test_error_deduplicated_per_db_label(self, tmp_path, caplog):
"""Repeated calls with the same db_label log exactly ONE error.
Prevents log spam when NFS users run kanban (which opens a fresh
connection on every operation — see hermes_cli/kanban_db.py).
Regression guard: the fix for #22032 ran apply_wal_with_fallback()
on every kb.connect() call; without dedup, errors.log fills with
hundreds of identical errors per hour.
"""
with caplog.at_level("ERROR", logger="hermes_state"):
# Three separate connections to "the same DB" via the same label
for i in range(3):
conn, _ = _open_blocking(tmp_path / f"dup-{i}.db", isolation_level=None)
mode = apply_wal_with_fallback(conn, db_label="shared.db")
assert mode == "delete"
conn.close()
# Exactly one error across all three calls
errors = [
r
for r in caplog.records
if r.levelname == "ERROR" and "shared.db" in r.getMessage()
]
assert len(errors) == 1, (
f"Expected 1 deduplicated error, got {len(errors)}: "
f"{[r.getMessage() for r in errors]}"
)
def test_falls_back_when_delete_pragma_also_fails(self, tmp_path, caplog):
"""WAL-incompat FS that ALSO rejects DELETE — must not crash callers (#30816).
``PRAGMA journal_mode=WAL`` raises a recognized WAL-incompat marker
(``locking protocol``), so the DELETE fallback engages — but DELETE
*also* raises (``disk I/O error``, observed on APFS external SSDs
under heavy contention). The connection's default journal_mode is
already DELETE, so it is still usable; propagating would crash
SessionDB / kanban_db / ResponseStore init. Returns ``"delete"``
and logs one WARNING per db_label.
Fault injection only: the APFS trigger itself is not reproducible on
Linux — this drives the same code path with a failing DELETE pragma.
"""
delete_attempts = [0]
class _DeletePragmaFailsConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
lowered = sql.lower().replace(" ", "")
if "journal_mode=wal" in lowered:
raise sqlite3.OperationalError("locking protocol")
if "journal_mode=delete" in lowered:
delete_attempts[0] += 1
raise sqlite3.OperationalError("disk I/O error")
return super().execute(sql, *args, **kwargs)
conn = sqlite3.connect(
str(tmp_path / "apfs.db"),
factory=_DeletePragmaFailsConnection,
isolation_level=None,
)
with caplog.at_level("WARNING", logger="hermes_state"):
mode = apply_wal_with_fallback(conn, db_label="apfs-test.db")
assert mode == "delete"
assert delete_attempts[0] == 1
msgs = [r.getMessage() for r in caplog.records if r.levelname in ("WARNING", "ERROR")]
assert any("apfs-test.db" in m and "WAL" in m for m in msgs)
delete_warnings = [
m for m in msgs if "apfs-test.db" in m and "both WAL and DELETE journal_mode failed" in m
]
assert len(delete_warnings) == 1
assert "disk I/O error" in delete_warnings[0]
# Connection is still usable for non-journal_mode SQL
conn.execute("CREATE TABLE t (x INTEGER)")
conn.execute("INSERT INTO t VALUES (1)")
assert list(conn.execute("SELECT x FROM t"))[0][0] == 1
conn.close()
def test_both_pragmas_fail_but_readback_reports_actual_mode(self, tmp_path, caplog):
"""When both PRAGMA writes fail, the return value is read back from the
connection — not a hardcoded ``"delete"`` guess.
The connection here already runs ``journal_mode=MEMORY`` (set before
the failure injection); both WAL and DELETE writes then fail, so the
function must report the actual ``"memory"`` mode.
"""
class _WritesFailReadsSucceedConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
lowered = sql.lower().replace(" ", "")
if "journal_mode=wal" in lowered:
raise sqlite3.OperationalError("locking protocol")
if "journal_mode=delete" in lowered:
raise sqlite3.OperationalError("disk I/O error")
return super().execute(sql, *args, **kwargs)
conn = sqlite3.connect(
str(tmp_path / "readback.db"),
factory=_WritesFailReadsSucceedConnection,
isolation_level=None,
)
conn.execute("PRAGMA journal_mode=MEMORY")
with caplog.at_level("WARNING", logger="hermes_state"):
mode = apply_wal_with_fallback(conn, db_label="readback.db")
assert mode == "memory"
msgs = [r.getMessage() for r in caplog.records if r.levelname in ("WARNING", "ERROR")]
assert any(
"readback.db" in m and "both WAL and DELETE journal_mode failed" in m
for m in msgs
)
conn.close()
def test_delete_fallback_failure_warning_deduplicated_per_db_label(self, tmp_path, caplog):
"""Repeated both-fail calls with the same db_label log exactly ONE WARNING.
kanban_db.connect() runs on every kanban operation; without dedup,
APFS-external-SSD users would see hundreds of identical warnings.
"""
class _BothPragmasFailConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
lowered = sql.lower().replace(" ", "")
if "journal_mode=wal" in lowered:
raise sqlite3.OperationalError("locking protocol")
if "journal_mode=delete" in lowered:
raise sqlite3.OperationalError("disk I/O error")
return super().execute(sql, *args, **kwargs)
with caplog.at_level("WARNING", logger="hermes_state"):
for i in range(3):
conn = sqlite3.connect(
str(tmp_path / f"apfs-dup-{i}.db"),
factory=_BothPragmasFailConnection,
isolation_level=None,
)
assert apply_wal_with_fallback(conn, db_label="apfs-shared.db") == "delete"
conn.close()
delete_warnings = [
r for r in caplog.records
if r.levelname == "WARNING"
and "apfs-shared.db" in r.getMessage()
and "both WAL and DELETE journal_mode failed" in r.getMessage()
]
assert len(delete_warnings) == 1, (
f"Expected 1 deduplicated DELETE-failed warning, got {len(delete_warnings)}"
)
def test_error_fires_independently_per_db_label(self, tmp_path, caplog):
"""Different db_labels each get their own one error (not globally dedup'd)."""
with caplog.at_level("ERROR", logger="hermes_state"):
conn1, _ = _open_blocking(tmp_path / "a.db", isolation_level=None)
apply_wal_with_fallback(conn1, db_label="state.db")
conn1.close()
conn2, _ = _open_blocking(tmp_path / "b.db", isolation_level=None)
apply_wal_with_fallback(conn2, db_label="kanban.db")
conn2.close()
errors = [r for r in caplog.records if r.levelname == "ERROR"]
labels_logged = {
lbl
for r in errors
for lbl in ("state.db", "kanban.db")
if lbl in r.getMessage()
}
assert labels_logged == {"state.db", "kanban.db"}, (
f"Each db_label should log once; got {labels_logged}"
)
class TestRequireWal:
"""``require_wal=True`` turns either WAL-refusal shape into a hard
``WalUnsupportedError`` for callers that mandate WAL concurrency, instead
of silently degrading to DELETE. The default callers (SessionDB /
kanban_db.connect) keep ``require_wal=False`` so NFS-homed installs work.
"""
def test_happy_path_unaffected_by_require_wal(self, tmp_path):
"""On a WAL-capable FS, require_wal=True still simply returns 'wal'."""
conn = sqlite3.connect(str(tmp_path / "ok.db"), isolation_level=None)
try:
assert apply_wal_with_fallback(conn, require_wal=True) == "wal"
finally:
conn.close()
def test_raises_on_raising_marker(self, tmp_path):
"""NFS-style raising refusal + require_wal=True → WalUnsupportedError."""
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
try:
with pytest.raises(WalUnsupportedError, match="locking protocol"):
apply_wal_with_fallback(conn, db_label="state.db", require_wal=True)
finally:
conn.close()
def test_raises_on_silent_refusal(self, tmp_path):
"""macOS-NFS silent no-op + require_wal=True → WalUnsupportedError,
not a false 'delete' return."""
factory = _make_silent_noop_factory("delete")
conn = sqlite3.connect(
str(tmp_path / "macnfs.db"), factory=factory, isolation_level=None
)
try:
with pytest.raises(WalUnsupportedError, match="refused without raising"):
apply_wal_with_fallback(conn, db_label="kanban.db", require_wal=True)
finally:
conn.close()
def test_error_is_operationalerror_subclass(self, tmp_path):
"""WalUnsupportedError must stay catchable as sqlite3.OperationalError
so existing DB-init handlers (e.g. SessionDB.__init__) still catch it."""
conn, _ = _open_blocking(tmp_path / "nfs.db", isolation_level=None)
try:
with pytest.raises(sqlite3.OperationalError):
apply_wal_with_fallback(conn, require_wal=True)
finally:
conn.close()
class TestGetLastInitError:
def test_captures_cause_on_failed_init(self, tmp_path):
"""When SessionDB() raises, the cause is preserved for slash commands.
Simulates a filesystem failure unrelated to the WAL fallback path —
e.g. ``foreign_keys=ON`` rejected by a read-only or seriously
damaged DB. (Both-pragmas-fail no longer raises since #30816: the
DELETE fallback is guarded and returns the mode in effect.) The
exception bubbles out of ``SessionDB.__init__`` and the cause is
captured for /resume to surface.
"""
target = tmp_path / "broken.db"
real_connect = sqlite3.connect
class _ForeignKeysFailConnection(sqlite3.Connection):
def execute(self, sql, *args, **kwargs): # type: ignore[override]
if "foreign_keys=on" in sql.lower().replace(" ", ""):
raise sqlite3.OperationalError(
"foreign_keys=ON rejected: read-only filesystem"
)
return super().execute(sql, *args, **kwargs)
def gated_connect(*args, **kwargs):
# connect_tracked passes a tracking-augmented factory; drop it and
# substitute the double, which connect_tracked will re-augment.
kwargs.pop("factory", None)
return real_connect(str(target), factory=_ForeignKeysFailConnection, **kwargs)
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
with pytest.raises(sqlite3.OperationalError):
SessionDB(db_path=target)
cause = get_last_init_error()
assert cause is not None
assert "OperationalError" in cause
assert "read-only filesystem" in cause
class TestSessionDbUsesWalFallback:
def test_sessiondb_works_when_wal_unavailable(self, tmp_path):
"""E2E: SessionDB initializes and performs a write on a WAL-blocked FS."""
target = tmp_path / "nfs_style.db"
real_connect = sqlite3.connect
attempts = [0]
factory = _make_blocking_factory("locking protocol", attempts)
def gated_connect(*args, **kwargs):
# connect_tracked passes a tracking-augmented factory; drop it and
# substitute the double, which connect_tracked re-applies to the
# returned instance.
kwargs.pop("factory", None)
return real_connect(str(target), factory=factory, **kwargs)
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
db = SessionDB(db_path=target)
try:
# WAL was attempted and rejected — fallback kicked in
assert attempts[0] >= 1, (
"WAL pragma was never executed — check the patch target"
)
# SessionDB is usable end-to-end: create a session, read it back
db.create_session(session_id="s1", source="cli", model="test")
sess = db.get_session("s1")
assert sess is not None
assert sess["source"] == "cli"
# No init error was recorded since init succeeded via the fallback
assert get_last_init_error() is None
finally:
db.close()
def test_sessiondb_works_when_wal_is_silently_refused(self, tmp_path, caplog):
"""E2E: SessionDB stays usable when WAL silently no-ops to DELETE."""
target = tmp_path / "macnfs_style.db"
real_connect = sqlite3.connect
factory = _make_silent_noop_factory("delete")
def gated_connect(*args, **kwargs):
# connect_tracked passes a tracking-augmented factory; drop it and
# substitute the double, which connect_tracked re-applies to the
# returned instance.
kwargs.pop("factory", None)
return real_connect(str(target), factory=factory, **kwargs)
with patch("hermes_state.sqlite3.connect", side_effect=gated_connect):
with caplog.at_level("ERROR", logger="hermes_state"):
db = SessionDB(db_path=target)
try:
assert db._conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "delete"
db.create_session(session_id="s2", source="cli", model="test")
sess = db.get_session("s2")
assert sess is not None
assert sess["source"] == "cli"
assert get_last_init_error() is None
finally:
db.close()
errors = [
r
for r in caplog.records
if r.levelname == "ERROR" and "state.db" in r.getMessage()
]
assert len(errors) == 1, (
"silent WAL refusal should emit exactly one state.db error"
)