import json from collections.abc import Iterator, Sequence from dataclasses import dataclass from typing import Final import pytest from datadog_reader import DdLogsReader from datadog_reader import _DdAuthHeaders # pyright: ignore[reportPrivateUsage] # verifies private auth-header serialization from e2e_config import DD_SEARCH_INTERVAL, POLL_TIMEOUT from e2e_http import StreamingResponse def test_failure_diagnostics_hide_credentials_without_changing_auth_headers() -> None: api_key: Final = "test-datadog-api-secret" app_key: Final = "test-datadog-app-secret" reader: Final = DdLogsReader(site="datadoghq.com", api_key=api_key, app_key=app_key) headers: Final = _DdAuthHeaders(api_key=api_key, app_key=app_key) for value in (reader, headers): assert api_key not in repr(value) assert app_key not in repr(value) assert headers.model_dump(by_alias=True) == { "DD-API-KEY": api_key, "DD-APPLICATION-KEY": app_key, } @dataclass class Clock: elapsed: float = 0.0 def now(self) -> float: return self.elapsed def sleep(self, seconds: float) -> None: self.elapsed += seconds @dataclass class Search: responses: Iterator[StreamingResponse] calls: tuple[tuple[str, float], ...] = () def __call__(self, query: str, timeout: float) -> StreamingResponse: self.calls += ((query, timeout),) return next(self.responses) def _page(*event_ids: str) -> StreamingResponse: return StreamingResponse( status_code=200, body=json.dumps({"data": [{"attributes": {"attributes": {"id": event_id}}} for event_id in event_ids]}), ) def _reader(responses: Sequence[StreamingResponse], clock: Clock) -> tuple[DdLogsReader, Search]: search: Final = Search(iter(responses)) return DdLogsReader( site="us5.datadoghq.com", api_key="test-api-secret", app_key="test-app-secret", search=search, now=clock.now, sleep=clock.sleep, jitter=lambda: 0.25, ), search def test_429_honors_server_reset_and_preserves_duplicate_events() -> None: clock: Final = Clock() reader, search = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "6"}), _page("first", "duplicate")), clock, ) events: Final = reader.events_for_query("test-marker") assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") assert clock.elapsed == 6.25 assert search.calls == (("test-marker", 30.0), ("test-marker", 30.0)) @pytest.mark.parametrize("reset", ("", "invalid", "nan", "inf", "-1")) def test_invalid_reset_uses_search_interval(reset: str) -> None: clock: Final = Clock() reader, _ = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": reset}), _page()), clock ) assert reader.events_for_query("test-marker") == [] assert clock.elapsed == DD_SEARCH_INTERVAL + 0.25 def test_zero_reset_cannot_create_a_busy_retry_loop() -> None: clock: Final = Clock() reader, _ = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "0"}), _page()), clock ) assert reader.events_for_query("test-marker") == [] assert clock.elapsed == 1.25 def test_retry_after_is_not_shortened_by_an_earlier_reset() -> None: clock: Final = Clock() reader, _ = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": "2", "retry-after": "8"}), _page()), clock, ) assert reader.events_for_query("test-marker") == [] assert clock.elapsed == 8.25 def test_rate_limit_wait_stops_at_deadline_without_issuing_another_request() -> None: clock: Final = Clock() reader, search = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT * 10)}),), clock ) with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): reader.events_for_query("test-marker") assert clock.elapsed == POLL_TIMEOUT assert search.calls == (("test-marker", 30.0),) def test_late_retry_cannot_receive_a_fresh_request_timeout() -> None: clock: Final = Clock() reader, search = _reader( (StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT - 5)}), _page()), clock, ) assert reader.events_for_query("test-marker") == [] assert search.calls == (("test-marker", 30.0), ("test-marker", 4.75)) @pytest.mark.parametrize("status", (-1, 401, 403, 500)) def test_non_quota_failures_are_not_retried_or_treated_as_empty_results(status: int) -> None: clock: Final = Clock() reader, search = _reader((StreamingResponse(status_code=status, body=""), _page()), clock) with pytest.raises(pytest.fail.Exception, match=f"failed with HTTP {status}"): reader.events_for_query("test-marker") assert search.calls == (("test-marker", 30.0),) assert clock.elapsed == 0 def test_polling_quota_retries_share_the_original_deadline() -> None: clock: Final = Clock() reader, search = _reader( (_page(), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), clock, ) with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): reader.poll_events_for_query("test-marker") assert clock.elapsed == POLL_TIMEOUT assert len(search.calls) == 2 def test_empty_polling_does_not_start_a_final_search_after_its_deadline() -> None: clock: Final = Clock() attempts: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) reader, search = _reader((_page(),) * attempts, clock) assert reader.poll_events_for_query("test-marker") == [] assert clock.elapsed == POLL_TIMEOUT assert len(search.calls) == attempts def test_settlement_quota_retries_keep_the_remaining_readback_budget() -> None: clock: Final = Clock() empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 reader, search = _reader( (_page(),) * empty_reads + (_page("first"), StreamingResponse(status_code=429, body="", headers={"x-ratelimit-reset": str(POLL_TIMEOUT)})), clock, ) with pytest.raises(pytest.fail.Exception, match="remained rate-limited"): reader.poll_events_for_query("test-marker") assert clock.elapsed == POLL_TIMEOUT assert search.calls[-1] == ("test-marker", DD_SEARCH_INTERVAL) assert len(search.calls) == empty_reads + 2 def test_settlement_detects_a_duplicate_on_the_final_search() -> None: clock: Final = Clock() reader, _ = _reader((_page("first"), _page("first"), _page(), _page("first", "duplicate")), clock) events: Final = reader.poll_events_for_query("test-marker") assert tuple(event.attributes["id"] for event in events) == ("first", "duplicate") assert clock.elapsed == 30 def test_settlement_keeps_confirmed_events_through_empty_searches() -> None: clock: Final = Clock() reader, _ = _reader((_page("first"), _page(), _page(), _page()), clock) events: Final = reader.poll_events_for_query("test-marker") assert tuple(event.attributes["id"] for event in events) == ("first",) assert clock.elapsed == 30 def test_late_delivery_cannot_pass_without_a_complete_settle_window() -> None: clock: Final = Clock() empty_reads: Final = int(POLL_TIMEOUT / DD_SEARCH_INTERVAL) - 2 reader, search = _reader((_page(),) * empty_reads + (_page("first"), _page("first")), clock) with pytest.raises(pytest.fail.Exception, match="duplicate-detection window"): reader.poll_events_for_query("test-marker") assert clock.elapsed == POLL_TIMEOUT assert len(search.calls) == empty_reads + 2