dns-blocklist-mirror/tests/test_mirroring.py
XLion 7d99d7e4e8
All checks were successful
Checks / checks (push) Successful in 45s
Deploy Pages / deploy (push) Successful in 20s
Split tests by responsibility
2026-08-15 22:34:31 +08:00

464 lines
14 KiB
Python

from __future__ import annotations
import socket
from pathlib import Path
import httpx
from dns_blocklist_mirror.cli import BuildPolicy, _mirror_all
from dns_blocklist_mirror.config import parse_config
from dns_blocklist_mirror.downloader import AttemptResult
from tests._support import Response, run_cli, serve_http, serve_https
def test_cli_uses_the_next_source_after_the_primary_fails(tmp_path: Path) -> None:
expected = b"fallback blocklist\n"
routes = {
"/primary.txt": Response(body=b"unavailable\n", status=404),
"/fallback.txt": Response(body=expected),
}
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/primary.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == expected
assert "hagezi/pro.txt: source 1/2 failed: HTTP 404 Not Found" in result.stdout
assert "hagezi/pro.txt: source 2/2 succeeded" in result.stdout
def test_cli_retries_a_transient_failure_before_using_fallback(tmp_path: Path) -> None:
primary_responses = [
Response(body=b"busy\n", status=503, headers={"Retry-After": "0"}),
Response(body=b"busy\n", status=503, headers={"Retry-After": "0"}),
Response(body=b"primary recovered\n"),
]
routes: dict[str, Response | list[Response]] = {
"/primary.txt": primary_responses,
"/fallback.txt": Response(body=b"fallback must not be used\n"),
}
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/primary.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"primary recovered\n"
assert primary_responses == []
assert (
"source 1/2 attempt 1/4 failed: HTTP 503 Service Unavailable" in result.stdout
)
assert (
"source 1/2 attempt 2/4 failed: HTTP 503 Service Unavailable" in result.stdout
)
assert "source 1/2 succeeded on attempt 3/4" in result.stdout
assert "source 2/2" not in result.stdout
def test_cli_ignores_an_invalid_retry_after_value(tmp_path: Path) -> None:
primary_responses = [
Response(body=b"busy\n", status=503, headers={"Retry-After": "NaN"}),
Response(body=b"primary recovered\n"),
]
with serve_https(tmp_path, {"/primary.txt": primary_responses}) as (
base_url,
ca_path,
):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/primary.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"primary recovered\n"
assert "Traceback" not in result.stderr
def test_cli_rejects_an_empty_response_and_uses_fallback(tmp_path: Path) -> None:
routes = {
"/empty.txt": Response(body=b"", headers={"Retry-After": "0"}),
"/fallback.txt": Response(body=b"complete list\n"),
}
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/empty.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"complete list\n"
assert result.stdout.count("source 1/2 attempt ") == 4
assert "source 1/2 attempt 1/4 failed: response body was empty" in result.stdout
assert "source 1/2 attempt 4/4 failed: response body was empty" in result.stdout
assert "source 2/2 succeeded" in result.stdout
def test_cli_rejects_a_response_larger_than_128_mib_before_reading_it(
tmp_path: Path,
) -> None:
routes = {
"/too-large.txt": Response(
body=b"x",
headers={"Content-Length": "134217729"},
),
"/fallback.txt": Response(body=b"bounded list\n"),
}
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/too-large.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"bounded list\n"
assert "source 1/2 failed: response exceeded the 128 MiB limit" in result.stdout
assert "source 2/2 succeeded" in result.stdout
def test_cli_rejects_an_https_redirect_that_downgrades_to_http(
tmp_path: Path,
) -> None:
insecure_responses = [Response(body=b"insecure content must not be requested\n")]
with serve_http({"/insecure.txt": insecure_responses}) as insecure_url:
routes = {
"/redirect.txt": Response(
body=b"",
status=302,
headers={"Location": f"{insecure_url}/insecure.txt"},
),
"/fallback.txt": Response(body=b"secure fallback\n"),
}
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/redirect.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"secure fallback\n"
assert insecure_responses == [
Response(body=b"insecure content must not be requested\n")
]
assert "source 1/2 failed: redirect downgraded from HTTPS to HTTP" in result.stdout
assert "source 2/2 succeeded" in result.stdout
def test_cli_follows_up_to_five_https_redirects(tmp_path: Path) -> None:
routes = {
f"/redirect-{index}.txt": Response(
body=b"",
status=302,
headers={"Location": f"/redirect-{index + 1}.txt"},
)
for index in range(5)
}
routes["/redirect-5.txt"] = Response(body=b"redirected list\n")
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/redirect-0.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"redirected list\n"
assert "source 1/1 succeeded" in result.stdout
def test_cli_uses_fallback_after_more_than_five_redirects(tmp_path: Path) -> None:
routes = {
f"/redirect-{index}.txt": Response(
body=b"",
status=302,
headers={"Location": f"/redirect-{index + 1}.txt"},
)
for index in range(6)
}
routes["/redirect-6.txt"] = Response(body=b"must not be reached\n")
routes["/fallback.txt"] = Response(body=b"bounded redirect fallback\n")
with serve_https(tmp_path, routes) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- {base_url}/redirect-0.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (
output_path / "hagezi" / "pro.txt"
).read_bytes() == b"bounded redirect fallback\n"
assert "source 1/2 failed: response exceeded the 5 redirect limit" in result.stdout
assert "source 2/2 succeeded" in result.stdout
def test_cli_retries_a_connection_failure_before_using_fallback(tmp_path: Path) -> None:
with socket.socket() as unavailable_socket:
unavailable_socket.bind(("127.0.0.1", 0))
unavailable_port = unavailable_socket.getsockname()[1]
with serve_https(
tmp_path,
{"/fallback.txt": Response(body=b"network fallback\n")},
) as (base_url, ca_path):
config_path = tmp_path / "mirrors.yaml"
config_path.write_text(
f"""\
version: 1
mirrors:
hagezi:
- destination: pro.txt
sources:
- https://127.0.0.1:{unavailable_port}/primary.txt
- {base_url}/fallback.txt
""",
encoding="utf-8",
)
output_path = tmp_path / "public"
result = run_cli(config_path, output_path, ca_path)
assert result.returncode == 0, result.stderr
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"network fallback\n"
assert "source 1/2 attempt 1/4 failed: connection failed:" in result.stdout
assert "source 1/2 attempt 4/4 failed: connection failed:" in result.stdout
assert "source 2/2 succeeded" in result.stdout
def test_build_deadline_prevents_another_attempt_after_retry_sleep(
tmp_path: Path,
) -> None:
config = parse_config(
{
"version": 1,
"mirrors": {
"hagezi": [
{
"destination": "pro.txt",
"sources": ["https://example.com/pro.txt"],
}
]
},
}
)
current_time = 0.0
attempt_timeouts: list[float] = []
sleeps: list[float] = []
def monotonic() -> float:
return current_time
def sleep(seconds: float) -> None:
nonlocal current_time
sleeps.append(seconds)
current_time += seconds
def download(
client: httpx.Client,
source: str,
destination: Path,
*,
attempt_timeout_seconds: float,
) -> AttemptResult:
attempt_timeouts.append(attempt_timeout_seconds)
return AttemptResult(
succeeded=False,
reason="HTTP 503 Service Unavailable",
retryable=True,
retry_after=60,
)
with httpx.Client() as client:
failures = _mirror_all(
config,
tmp_path / "staging",
client,
policy=BuildPolicy(
max_retries=3,
max_retry_delay_seconds=30,
build_timeout_seconds=2,
attempt_timeout_seconds=5,
),
monotonic=monotonic,
sleep=sleep,
download=download,
)
assert attempt_timeouts == [2]
assert sleeps == [2]
assert failures == [
(
Path("hagezi/pro.txt"),
[
(
"https://example.com/pro.txt",
"build exceeded the 30 minute limit",
)
],
)
]
def test_retry_policy_uses_three_retries_and_exponential_backoff(
tmp_path: Path,
) -> None:
config = parse_config(
{
"version": 1,
"mirrors": {
"hagezi": [
{
"destination": "pro.txt",
"sources": ["https://example.com/pro.txt"],
}
]
},
}
)
current_time = 0.0
attempts = 0
sleeps: list[float] = []
def monotonic() -> float:
return current_time
def sleep(seconds: float) -> None:
nonlocal current_time
sleeps.append(seconds)
current_time += seconds
def download(
client: httpx.Client,
source: str,
destination: Path,
*,
attempt_timeout_seconds: float,
) -> AttemptResult:
nonlocal attempts
attempts += 1
return AttemptResult(
succeeded=False,
reason="HTTP 503 Service Unavailable",
retryable=True,
)
with httpx.Client() as client:
failures = _mirror_all(
config,
tmp_path / "staging",
client,
policy=BuildPolicy(build_timeout_seconds=20),
monotonic=monotonic,
sleep=sleep,
download=download,
)
assert attempts == 4
assert sleeps == [1, 2, 4]
assert failures == [
(
Path("hagezi/pro.txt"),
[("https://example.com/pro.txt", "HTTP 503 Service Unavailable")],
)
]