litellm/tests/e2e/logging/test_s3_log_e2e.py

"""Live e2e: s3_v2 log delivery for successful and failed calls.

Covers logging.s3.success.writes_object and logging.s3.failure.writes_object:
one /chat/completions call must land in the real S3 bucket as exactly one
StandardLoggingPayload object (the primary audit trail; the batch flush must
neither drop nor duplicate it), and a failed call must be persisted the same
way for compliance. Delivery is judged on what is actually in the bucket: the
proxy writes with its production credentials and the test lists and reads the
objects back.

Both halves of the contract are asserted: the recorded state (the proxy
reports the S3Logger callback active via /health/readiness/details) and the
enforced behavior (the object in the bucket, with the cost cross-checked
against the x-litellm-response-cost header of the very response the caller
received).

The suite requires ``s3_callback_params.s3_use_key_prefix: true`` on the proxy,
which keys objects as ``{key_alias}/{date}/time-..._{id}.json`` - a unique key
alias per test turns the poll into a cheap prefix listing.
"""

from __future__ import annotations

import math
import time

import pytest

from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from lifecycle import ResourceManager
from logging_client import (
    INVALID_UPSTREAM_API_KEY,
    LoggingClient,
    completion_response_id,
    first_ok,
    readiness_details_body,
)
from models import LiteLLMParamsBody
from s3_reader import S3LogReader, build_s3_reader

pytestmark = pytest.mark.e2e

#: The active s3_v2 callback's name in /health/readiness/details success_callbacks.
S3_LOGGER_NAME = "S3Logger"


@pytest.fixture(scope="session")
def s3_logs() -> S3LogReader:
    return build_s3_reader()


def _assert_s3_configured(client: LoggingClient) -> None:
    """Recorded state: the proxy reports the s3_v2 callback among its active
    callbacks, so a missing destination config fails here, before any
    delivery-based assertion can time out confusingly."""
    body = readiness_details_body(client)
    assert S3_LOGGER_NAME in body, (
        f"the proxy must report the {S3_LOGGER_NAME} callback active "
        f"(litellm_settings.callbacks: ['s3_v2'] + s3_callback_params in the proxy config); "
        f"got: {body[:400]}"
    )


class TestS3LogDelivery:
    @pytest.mark.covers("logging.s3.success.writes_object", exercised_on=["chat_completions"])
    def test_chat_completions_writes_one_success_object(
        self, client: LoggingClient, s3_logs: S3LogReader, resources: ResourceManager
    ) -> None:
        """One successful non-streaming /chat/completions call must land in
        the bucket as exactly one payload object carrying the model group, the
        token counts, and the same cost the caller's response header reported."""
        _assert_s3_configured(client)

        alias = f"s3-chat-{unique_marker()}"
        key = client.key_with_alias(alias, models=[CHEAP_ANTHROPIC_MODEL])
        resources.defer(lambda: client.delete_key(key))

        marker = unique_marker()
        outcome = first_ok(
            client,
            lambda: client.chat_raw(key, CHEAP_ANTHROPIC_MODEL, f"reply with one word {marker}", max_tokens=16),
        )
        assert outcome.response_cost is not None and outcome.response_cost > 0, (
            f"the response must report x-litellm-response-cost, got {outcome.response_cost!r}"
        )
        body_id = completion_response_id(outcome.body)
        assert body_id is not None, "the completion body must carry an id (it names the s3 object)"

        records = s3_logs.poll_records(prefix=f"{alias}/", predicate=lambda r: r.id == body_id)
        assert records, (
            f"no s3 object for response {body_id} under prefix {alias}/ reached the bucket within the deadline"
        )
        assert len(records) == 1, (
            f"expected exactly ONE s3 object for the call, got {len(records)} - "
            "more than one object for one call is the duplicate-delivery bug"
        )
        record = records[0]
        assert record.status == "success", f"payload status must be success, got {record.status!r}"
        assert record.model_group == CHEAP_ANTHROPIC_MODEL, (
            f"payload model_group must be {CHEAP_ANTHROPIC_MODEL!r}, got {record.model_group!r}"
        )
        assert record.total_tokens is not None and record.total_tokens > 0, (
            f"payload must count real tokens, got {record.total_tokens!r}"
        )
        assert record.response_cost is not None and math.isclose(
            record.response_cost, outcome.response_cost, rel_tol=1e-9
        ), f"payload response_cost {record.response_cost!r} must equal the header cost {outcome.response_cost}"

    @pytest.mark.covers("logging.s3.failure.writes_object", exercised_on=["chat_completions"])
    def test_chat_completions_failure_writes_one_object(
        self, client: LoggingClient, s3_logs: S3LogReader, resources: ResourceManager
    ) -> None:
        """A call that fails at the provider must be persisted to the bucket as
        exactly one failure payload carrying the provider error - failed calls
        are part of the audit trail, not an exemption from it.

        A deployment with an invalid upstream key lets the request pass proxy
        auth and fail at the provider (the same lever as the OTEL error test).
        Proxy-side rejections during key/model propagation can also ship
        failure payloads under this alias, but without a model_group and
        without the provider error, so the read-back keys on both: only
        provider-reaching calls carry them, and with this key every one of
        those is the AnthropicException that ends the send loop."""
        _assert_s3_configured(client)

        model_name = f"s3-err-{unique_marker()}"
        model_id = client.create_model(
            model_name,
            LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY),
        )
        resources.defer(lambda: client.delete_model(model_id))
        alias = f"s3-err-key-{unique_marker()}"
        key = client.key_with_alias(alias, models=[model_name])
        resources.defer(lambda: client.delete_key(key))

        deadline = time.monotonic() + client.proxy.poll_timeout
        while True:
            outcome = client.chat_raw(key, model_name, "trigger an upstream auth failure", max_tokens=16)
            assert not outcome.ok, "the call must fail; the deployment's upstream key is invalid"
            assert outcome.status_code != -1, (
                "network failure between the test and the proxy while provoking the provider "
                "failure; retrying now could double-log the failure payload and falsely trip "
                f"the exactly-one assertion - fix the rig connectivity first: {outcome.body[:200]}"
            )
            if "AnthropicException" in outcome.body or time.monotonic() >= deadline:
                break
            time.sleep(client.proxy.poll_interval)
        assert "AnthropicException" in outcome.body, (
            "never saw the upstream provider failure before the deadline; the key may still be "
            f"propagating - last outcome {outcome.status_code}: {outcome.body[:200]}"
        )
        assert outcome.status_code == 401, (
            f"an upstream auth failure must map to 401, got {outcome.status_code}: {outcome.body[:200]}"
        )

        records = s3_logs.poll_records(
            prefix=f"{alias}/",
            predicate=lambda r: (
                r.status == "failure" and r.model_group == model_name and "AnthropicException" in (r.error_str or "")
            ),
        )
        assert records, (
            f"no failure object for {model_name} under prefix {alias}/ reached the bucket within the deadline"
        )
        assert len(records) == 1, f"expected exactly ONE failure object for the call, got {len(records)}"
        record = records[0]
        assert record.error_str is not None and "AnthropicException" in record.error_str, (
            f"the persisted failure must carry the provider error, got error_str={record.error_str!r}"
        )
        assert not record.response_cost, f"a failed call must not be billed, got response_cost={record.response_cost!r}"