464 lines
14 KiB
Python
464 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import socket
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from dns_blocklist_mirror.cli import BuildPolicy, _mirror_all
|
|
from dns_blocklist_mirror.config import parse_config
|
|
from dns_blocklist_mirror.downloader import AttemptResult
|
|
from tests._support import Response, run_cli, serve_http, serve_https
|
|
|
|
|
|
def test_cli_uses_the_next_source_after_the_primary_fails(tmp_path: Path) -> None:
|
|
expected = b"fallback blocklist\n"
|
|
routes = {
|
|
"/primary.txt": Response(body=b"unavailable\n", status=404),
|
|
"/fallback.txt": Response(body=expected),
|
|
}
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/primary.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == expected
|
|
assert "hagezi/pro.txt: source 1/2 failed: HTTP 404 Not Found" in result.stdout
|
|
assert "hagezi/pro.txt: source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_retries_a_transient_failure_before_using_fallback(tmp_path: Path) -> None:
|
|
primary_responses = [
|
|
Response(body=b"busy\n", status=503, headers={"Retry-After": "0"}),
|
|
Response(body=b"busy\n", status=503, headers={"Retry-After": "0"}),
|
|
Response(body=b"primary recovered\n"),
|
|
]
|
|
routes: dict[str, Response | list[Response]] = {
|
|
"/primary.txt": primary_responses,
|
|
"/fallback.txt": Response(body=b"fallback must not be used\n"),
|
|
}
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/primary.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"primary recovered\n"
|
|
assert primary_responses == []
|
|
assert (
|
|
"source 1/2 attempt 1/4 failed: HTTP 503 Service Unavailable" in result.stdout
|
|
)
|
|
assert (
|
|
"source 1/2 attempt 2/4 failed: HTTP 503 Service Unavailable" in result.stdout
|
|
)
|
|
assert "source 1/2 succeeded on attempt 3/4" in result.stdout
|
|
assert "source 2/2" not in result.stdout
|
|
|
|
|
|
def test_cli_ignores_an_invalid_retry_after_value(tmp_path: Path) -> None:
|
|
primary_responses = [
|
|
Response(body=b"busy\n", status=503, headers={"Retry-After": "NaN"}),
|
|
Response(body=b"primary recovered\n"),
|
|
]
|
|
|
|
with serve_https(tmp_path, {"/primary.txt": primary_responses}) as (
|
|
base_url,
|
|
ca_path,
|
|
):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/primary.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"primary recovered\n"
|
|
assert "Traceback" not in result.stderr
|
|
|
|
|
|
def test_cli_rejects_an_empty_response_and_uses_fallback(tmp_path: Path) -> None:
|
|
routes = {
|
|
"/empty.txt": Response(body=b"", headers={"Retry-After": "0"}),
|
|
"/fallback.txt": Response(body=b"complete list\n"),
|
|
}
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/empty.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"complete list\n"
|
|
assert result.stdout.count("source 1/2 attempt ") == 4
|
|
assert "source 1/2 attempt 1/4 failed: response body was empty" in result.stdout
|
|
assert "source 1/2 attempt 4/4 failed: response body was empty" in result.stdout
|
|
assert "source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_rejects_a_response_larger_than_128_mib_before_reading_it(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
routes = {
|
|
"/too-large.txt": Response(
|
|
body=b"x",
|
|
headers={"Content-Length": "134217729"},
|
|
),
|
|
"/fallback.txt": Response(body=b"bounded list\n"),
|
|
}
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/too-large.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"bounded list\n"
|
|
assert "source 1/2 failed: response exceeded the 128 MiB limit" in result.stdout
|
|
assert "source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_rejects_an_https_redirect_that_downgrades_to_http(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
insecure_responses = [Response(body=b"insecure content must not be requested\n")]
|
|
with serve_http({"/insecure.txt": insecure_responses}) as insecure_url:
|
|
routes = {
|
|
"/redirect.txt": Response(
|
|
body=b"",
|
|
status=302,
|
|
headers={"Location": f"{insecure_url}/insecure.txt"},
|
|
),
|
|
"/fallback.txt": Response(body=b"secure fallback\n"),
|
|
}
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/redirect.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"secure fallback\n"
|
|
assert insecure_responses == [
|
|
Response(body=b"insecure content must not be requested\n")
|
|
]
|
|
assert "source 1/2 failed: redirect downgraded from HTTPS to HTTP" in result.stdout
|
|
assert "source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_follows_up_to_five_https_redirects(tmp_path: Path) -> None:
|
|
routes = {
|
|
f"/redirect-{index}.txt": Response(
|
|
body=b"",
|
|
status=302,
|
|
headers={"Location": f"/redirect-{index + 1}.txt"},
|
|
)
|
|
for index in range(5)
|
|
}
|
|
routes["/redirect-5.txt"] = Response(body=b"redirected list\n")
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/redirect-0.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"redirected list\n"
|
|
assert "source 1/1 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_uses_fallback_after_more_than_five_redirects(tmp_path: Path) -> None:
|
|
routes = {
|
|
f"/redirect-{index}.txt": Response(
|
|
body=b"",
|
|
status=302,
|
|
headers={"Location": f"/redirect-{index + 1}.txt"},
|
|
)
|
|
for index in range(6)
|
|
}
|
|
routes["/redirect-6.txt"] = Response(body=b"must not be reached\n")
|
|
routes["/fallback.txt"] = Response(body=b"bounded redirect fallback\n")
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- {base_url}/redirect-0.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (
|
|
output_path / "hagezi" / "pro.txt"
|
|
).read_bytes() == b"bounded redirect fallback\n"
|
|
assert "source 1/2 failed: response exceeded the 5 redirect limit" in result.stdout
|
|
assert "source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_cli_retries_a_connection_failure_before_using_fallback(tmp_path: Path) -> None:
|
|
with socket.socket() as unavailable_socket:
|
|
unavailable_socket.bind(("127.0.0.1", 0))
|
|
unavailable_port = unavailable_socket.getsockname()[1]
|
|
|
|
with serve_https(
|
|
tmp_path,
|
|
{"/fallback.txt": Response(body=b"network fallback\n")},
|
|
) as (base_url, ca_path):
|
|
config_path = tmp_path / "mirrors.yaml"
|
|
config_path.write_text(
|
|
f"""\
|
|
version: 1
|
|
mirrors:
|
|
hagezi:
|
|
- destination: pro.txt
|
|
sources:
|
|
- https://127.0.0.1:{unavailable_port}/primary.txt
|
|
- {base_url}/fallback.txt
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
output_path = tmp_path / "public"
|
|
|
|
result = run_cli(config_path, output_path, ca_path)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert (output_path / "hagezi" / "pro.txt").read_bytes() == b"network fallback\n"
|
|
assert "source 1/2 attempt 1/4 failed: connection failed:" in result.stdout
|
|
assert "source 1/2 attempt 4/4 failed: connection failed:" in result.stdout
|
|
assert "source 2/2 succeeded" in result.stdout
|
|
|
|
|
|
def test_build_deadline_prevents_another_attempt_after_retry_sleep(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
config = parse_config(
|
|
{
|
|
"version": 1,
|
|
"mirrors": {
|
|
"hagezi": [
|
|
{
|
|
"destination": "pro.txt",
|
|
"sources": ["https://example.com/pro.txt"],
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
current_time = 0.0
|
|
attempt_timeouts: list[float] = []
|
|
sleeps: list[float] = []
|
|
|
|
def monotonic() -> float:
|
|
return current_time
|
|
|
|
def sleep(seconds: float) -> None:
|
|
nonlocal current_time
|
|
sleeps.append(seconds)
|
|
current_time += seconds
|
|
|
|
def download(
|
|
client: httpx.Client,
|
|
source: str,
|
|
destination: Path,
|
|
*,
|
|
attempt_timeout_seconds: float,
|
|
) -> AttemptResult:
|
|
attempt_timeouts.append(attempt_timeout_seconds)
|
|
return AttemptResult(
|
|
succeeded=False,
|
|
reason="HTTP 503 Service Unavailable",
|
|
retryable=True,
|
|
retry_after=60,
|
|
)
|
|
|
|
with httpx.Client() as client:
|
|
failures = _mirror_all(
|
|
config,
|
|
tmp_path / "staging",
|
|
client,
|
|
policy=BuildPolicy(
|
|
max_retries=3,
|
|
max_retry_delay_seconds=30,
|
|
build_timeout_seconds=2,
|
|
attempt_timeout_seconds=5,
|
|
),
|
|
monotonic=monotonic,
|
|
sleep=sleep,
|
|
download=download,
|
|
)
|
|
|
|
assert attempt_timeouts == [2]
|
|
assert sleeps == [2]
|
|
assert failures == [
|
|
(
|
|
Path("hagezi/pro.txt"),
|
|
[
|
|
(
|
|
"https://example.com/pro.txt",
|
|
"build exceeded the 30 minute limit",
|
|
)
|
|
],
|
|
)
|
|
]
|
|
|
|
|
|
def test_retry_policy_uses_three_retries_and_exponential_backoff(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
config = parse_config(
|
|
{
|
|
"version": 1,
|
|
"mirrors": {
|
|
"hagezi": [
|
|
{
|
|
"destination": "pro.txt",
|
|
"sources": ["https://example.com/pro.txt"],
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
current_time = 0.0
|
|
attempts = 0
|
|
sleeps: list[float] = []
|
|
|
|
def monotonic() -> float:
|
|
return current_time
|
|
|
|
def sleep(seconds: float) -> None:
|
|
nonlocal current_time
|
|
sleeps.append(seconds)
|
|
current_time += seconds
|
|
|
|
def download(
|
|
client: httpx.Client,
|
|
source: str,
|
|
destination: Path,
|
|
*,
|
|
attempt_timeout_seconds: float,
|
|
) -> AttemptResult:
|
|
nonlocal attempts
|
|
attempts += 1
|
|
return AttemptResult(
|
|
succeeded=False,
|
|
reason="HTTP 503 Service Unavailable",
|
|
retryable=True,
|
|
)
|
|
|
|
with httpx.Client() as client:
|
|
failures = _mirror_all(
|
|
config,
|
|
tmp_path / "staging",
|
|
client,
|
|
policy=BuildPolicy(build_timeout_seconds=20),
|
|
monotonic=monotonic,
|
|
sleep=sleep,
|
|
download=download,
|
|
)
|
|
|
|
assert attempts == 4
|
|
assert sleeps == [1, 2, 4]
|
|
assert failures == [
|
|
(
|
|
Path("hagezi/pro.txt"),
|
|
[("https://example.com/pro.txt", "HTTP 503 Service Unavailable")],
|
|
)
|
|
]
|