244 lines
7.3 KiB
Python
244 lines
7.3 KiB
Python
from __future__ import annotations
|
|
|
|
import signal
|
|
import ssl
|
|
import time
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from dns_blocklist_mirror.downloader import _wall_clock_deadline, download_once
|
|
from tests._support import Response, serve_https
|
|
|
|
|
|
def test_wall_clock_deadline_restores_an_existing_timer(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
import dns_blocklist_mirror.downloader as downloader
|
|
|
|
timer_calls: list[tuple[int, float, float]] = []
|
|
handler_calls: list[tuple[int, object]] = []
|
|
clock_values = iter((100.0, 103.0))
|
|
previous_handler = object()
|
|
|
|
monkeypatch.setattr(
|
|
downloader.signal,
|
|
"getitimer",
|
|
lambda timer: (10.0, 2.0),
|
|
)
|
|
monkeypatch.setattr(
|
|
downloader.signal,
|
|
"getsignal",
|
|
lambda signal_number: previous_handler,
|
|
)
|
|
monkeypatch.setattr(
|
|
downloader.signal,
|
|
"signal",
|
|
lambda signal_number, handler: handler_calls.append((signal_number, handler)),
|
|
)
|
|
monkeypatch.setattr(
|
|
downloader.signal,
|
|
"setitimer",
|
|
lambda timer, delay, interval=0.0: timer_calls.append((timer, delay, interval)),
|
|
)
|
|
monkeypatch.setattr(downloader.time, "monotonic", lambda: next(clock_values))
|
|
|
|
with _wall_clock_deadline(5):
|
|
pass
|
|
|
|
assert timer_calls == [
|
|
(signal.ITIMER_REAL, 5, 0),
|
|
(signal.ITIMER_REAL, 0, 0),
|
|
(signal.ITIMER_REAL, 7, 2),
|
|
]
|
|
assert handler_calls[0][0] == signal.SIGALRM
|
|
assert handler_calls[1] == (signal.SIGALRM, previous_handler)
|
|
|
|
|
|
def test_streaming_download_enforces_the_limit_without_content_length(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
routes = {
|
|
"/chunked.txt": Response(
|
|
body=b"123456789",
|
|
include_content_length=False,
|
|
)
|
|
}
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
tls_context = ssl.create_default_context(cafile=ca_path)
|
|
with httpx.Client(verify=tls_context, follow_redirects=False) as client:
|
|
result = download_once(
|
|
client,
|
|
f"{base_url}/chunked.txt",
|
|
tmp_path / "output.txt",
|
|
max_file_bytes=8,
|
|
)
|
|
|
|
assert not result.succeeded
|
|
assert result.reason == "response exceeded the 8-byte limit"
|
|
assert (tmp_path / "output.txt").exists()
|
|
|
|
|
|
def test_download_retries_by_truncating_the_staging_destination(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
routes = {
|
|
"/pro.txt": [
|
|
Response(
|
|
body=b"stale content",
|
|
headers={"Content-Length": "64"},
|
|
),
|
|
Response(body=b"fresh content"),
|
|
]
|
|
}
|
|
destination = tmp_path / "output.txt"
|
|
|
|
with serve_https(tmp_path, routes) as (base_url, ca_path):
|
|
tls_context = ssl.create_default_context(cafile=ca_path)
|
|
with httpx.Client(verify=tls_context, follow_redirects=False) as client:
|
|
first_result = download_once(
|
|
client,
|
|
f"{base_url}/pro.txt",
|
|
destination,
|
|
)
|
|
second_result = download_once(
|
|
client,
|
|
f"{base_url}/pro.txt",
|
|
destination,
|
|
)
|
|
|
|
assert not first_result.succeeded
|
|
assert first_result.retryable
|
|
assert second_result.succeeded
|
|
assert destination.read_bytes() == b"fresh content"
|
|
|
|
|
|
def test_download_reports_a_decoding_error_without_a_traceback(tmp_path: Path) -> None:
|
|
def malformed_gzip(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
headers={"Content-Encoding": "gzip"},
|
|
content=b"not a gzip stream",
|
|
)
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(malformed_gzip)) as client:
|
|
result = download_once(
|
|
client,
|
|
"https://example.com/pro.txt",
|
|
tmp_path / "output.txt",
|
|
)
|
|
|
|
assert not result.succeeded
|
|
assert not result.retryable
|
|
assert result.reason == "response body could not be decoded"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("retry_after", "expected"),
|
|
[
|
|
("NaN", None),
|
|
("Fri, 31 Dec 2099 23:59:59 GMT", 30),
|
|
],
|
|
)
|
|
def test_download_parses_retry_after_safely(
|
|
tmp_path: Path,
|
|
retry_after: str,
|
|
expected: float | None,
|
|
) -> None:
|
|
def unavailable(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(503, headers={"Retry-After": retry_after})
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(unavailable)) as client:
|
|
result = download_once(
|
|
client,
|
|
"https://example.com/pro.txt",
|
|
tmp_path / "output.txt",
|
|
)
|
|
|
|
if expected is None:
|
|
assert result.retry_after is None
|
|
else:
|
|
assert result.retry_after is not None
|
|
assert result.retry_after > expected
|
|
|
|
|
|
def test_download_applies_the_remaining_attempt_timeout(tmp_path: Path) -> None:
|
|
request_timeouts: list[dict[str, float]] = []
|
|
|
|
def respond(request: httpx.Request) -> httpx.Response:
|
|
request_timeouts.append(request.extensions["timeout"])
|
|
return httpx.Response(200, content=b"list\n")
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(respond)) as client:
|
|
result = download_once(
|
|
client,
|
|
"https://example.com/pro.txt",
|
|
tmp_path / "output.txt",
|
|
attempt_timeout_seconds=1.25,
|
|
monotonic=lambda: 0.0,
|
|
)
|
|
|
|
assert result.succeeded
|
|
assert request_timeouts == [
|
|
{"connect": 1.25, "read": 1.25, "write": 1.25, "pool": 1.25}
|
|
]
|
|
|
|
|
|
def test_download_rejects_a_stream_that_reaches_eof_after_the_deadline(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
current_time = 0.0
|
|
|
|
class LateEofStream(httpx.SyncByteStream):
|
|
def __iter__(self) -> Iterator[bytes]:
|
|
nonlocal current_time
|
|
yield b"list\n"
|
|
current_time = 6
|
|
|
|
def respond(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, stream=LateEofStream())
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(respond)) as client:
|
|
result = download_once(
|
|
client,
|
|
"https://example.com/pro.txt",
|
|
tmp_path / "output.txt",
|
|
attempt_timeout_seconds=5,
|
|
monotonic=lambda: current_time,
|
|
)
|
|
|
|
assert not result.succeeded
|
|
assert result.retryable
|
|
assert result.reason == "download exceeded the per-attempt time limit"
|
|
assert (tmp_path / "output.txt").exists()
|
|
|
|
|
|
def test_download_interrupts_a_blocking_stream_at_the_attempt_deadline(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
class SlowStream(httpx.SyncByteStream):
|
|
def __iter__(self) -> Iterator[bytes]:
|
|
time.sleep(0.5)
|
|
yield b"too late\n"
|
|
|
|
def respond(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, stream=SlowStream())
|
|
|
|
started_at = time.monotonic()
|
|
with httpx.Client(transport=httpx.MockTransport(respond)) as client:
|
|
result = download_once(
|
|
client,
|
|
"https://example.com/pro.txt",
|
|
tmp_path / "output.txt",
|
|
attempt_timeout_seconds=0.05,
|
|
)
|
|
elapsed = time.monotonic() - started_at
|
|
|
|
assert not result.succeeded
|
|
assert result.retryable
|
|
assert result.reason == "download exceeded the per-attempt time limit"
|
|
assert elapsed < 0.25
|
|
assert (tmp_path / "output.txt").exists()
|