diff --git a/cookbook/logging_observability/hex_spend_dashboard.hex.yaml b/cookbook/logging_observability/hex_spend_dashboard.hex.yaml new file mode 100644 index 00000000000..6a5d9a9f2b3 --- /dev/null +++ b/cookbook/logging_observability/hex_spend_dashboard.hex.yaml @@ -0,0 +1,239 @@ +schemaVersion: 3 +meta: + title: LiteLLM spend analytics + description: Request spend, token usage, errors, models, and teams from LiteLLM via S3 and Athena. + codeLanguage: PYTHON + timezone: UTC + autoRerunApp: true + logicQueryCacheTimeout: + type: disabled + publishedQueryCacheTimeout: + type: disabled +cells: +- cellId: bb3d1f25-5065-58b9-bca1-049cf32398de + cellLabel: Setup + cellType: MARKDOWN + config: + source: |- + # LiteLLM spend analytics + + 1. Enable `litellm_settings: {callbacks: [hex]}` on the proxy. Set `HEX_S3_BUCKET_NAME`, optionally `HEX_S3_REGION_NAME` and `HEX_S3_PATH` (default `litellm-spend`). Use the proxy's AWS IAM role to write logs. + 2. Run the accompanying `hex_spend_table.sql` in Athena after replacing its bucket and prefix. Partition projection discovers new daily folders without a crawler or partition repair. Keep query results outside the log prefix. + 3. Add an [Athena data connection](https://learn.hex.tech/docs/connect-to-data/data-connections/setup-guides/connect-to-amazon-athena) to Hex, preferably using an IAM role. Select that connection for the four SQL cells below and choose the database containing `litellm_hex_spend`. + 4. Run all cells, publish the App, and configure a Hex scheduled run for your desired refresh interval. Refresh queries read newly uploaded records; this is not a live streaming dashboard. + + This template covers the last 30 UTC calendar days, including today. Spend is in USD. Requests with unknown cost are counted separately rather than assumed free. No prompts, responses, raw errors, or API keys are exported. Team/user identifiers and key aliases are included; control access to the bucket and Hex project accordingly. + + Uploads are buffered in process and retried by the S3 logger. A proxy crash can lose records that have not reached S3. Use your existing warehouse pipeline and native Hex connection instead if you already have one; this callback does not require a Hex API key. +- cellId: b38ace16-60ff-5713-9fd4-c2afac69e732 + cellLabel: Summary query + cellType: SQL + config: + dataFrameCell: false + dataConnectionId: null + resultVariableName: spend_summary + source: |- + SELECT COUNT(*) AS requests, + COALESCE(SUM(spend), 0) AS total_spend, + COUNT(*) FILTER (WHERE spend IS NULL) AS unknown_cost_requests, + COALESCE(SUM(total_tokens), 0) AS tokens, + COUNT(*) FILTER (WHERE status = 'failure') AS failed_requests, + AVG(response_time) AS average_latency_seconds + FROM litellm_hex_spend + WHERE log_date BETWEEN CAST(date_add('day', -29, current_date) AS varchar) + AND CAST(current_date AS varchar) +- cellId: b1ff3cf7-66ca-5013-90a0-5b7368c52773 + cellLabel: Daily spend query + cellType: SQL + config: + dataFrameCell: false + dataConnectionId: null + resultVariableName: daily_spend + source: |- + SELECT log_date, COALESCE(SUM(spend), 0) AS spend, + COUNT(*) AS requests + FROM litellm_hex_spend + WHERE log_date BETWEEN CAST(date_add('day', -29, current_date) AS varchar) + AND CAST(current_date AS varchar) + GROUP BY log_date + ORDER BY log_date +- cellId: 08d70a66-b6f8-587c-9bcf-b71e639c4239 + cellLabel: Model spend query + cellType: SQL + config: + dataFrameCell: false + dataConnectionId: null + resultVariableName: model_spend + source: |- + SELECT COALESCE(model_group, model, '(unassigned)') AS model, + COALESCE(SUM(spend), 0) AS spend, COUNT(*) AS requests, + COALESCE(SUM(total_tokens), 0) AS tokens + FROM litellm_hex_spend + WHERE log_date BETWEEN CAST(date_add('day', -29, current_date) AS varchar) + AND CAST(current_date AS varchar) + GROUP BY 1 + ORDER BY spend DESC +- cellId: 5199f238-552a-5d14-8368-e83ce7414369 + cellLabel: Team spend query + cellType: SQL + config: + dataFrameCell: false + dataConnectionId: null + resultVariableName: team_spend + source: |- + SELECT COALESCE(team_id, '(unassigned)') AS team_id, + COALESCE(SUM(spend), 0) AS spend, COUNT(*) AS requests, + COUNT(*) FILTER (WHERE status = 'failure') AS failed_requests, + AVG(response_time) AS average_latency_seconds + FROM litellm_hex_spend + WHERE log_date BETWEEN CAST(date_add('day', -29, current_date) AS varchar) + AND CAST(current_date AS varchar) + GROUP BY 1 + ORDER BY spend DESC +- cellId: ab06dca2-ee5d-5b43-99f4-13dbbd6f0404 + cellLabel: Spend (USD) + cellType: METRIC + config: + title: Spend (USD) + valueColumn: total_spend + valueRowIndex: 0 + valueVariableName: spend_summary + showComparison: false +- cellId: fc77975b-89e4-58de-ae86-846c534d2c25 + cellLabel: Requests + cellType: METRIC + config: + title: Requests + valueColumn: requests + valueRowIndex: 0 + valueVariableName: spend_summary + showComparison: false +- cellId: 185f508e-ece0-5b72-9052-a27042ce4bdd + cellLabel: Tokens + cellType: METRIC + config: + title: Tokens + valueColumn: tokens + valueRowIndex: 0 + valueVariableName: spend_summary + showComparison: false +- cellId: 856624aa-0a40-5c27-99e9-873c98b16d4d + cellLabel: Failed requests + cellType: METRIC + config: + title: Failed requests + valueColumn: failed_requests + valueRowIndex: 0 + valueVariableName: spend_summary + showComparison: false +- cellId: ae69da31-e28b-5ace-b622-4376380822cb + cellLabel: Unknown cost requests + cellType: METRIC + config: + title: Unknown cost requests + valueColumn: unknown_cost_requests + valueRowIndex: 0 + valueVariableName: spend_summary + showComparison: false +- cellId: 55436d1d-3d4e-5068-b192-d540d488d9d0 + cellLabel: Daily spend + cellType: CODE + config: + source: |- + import plotly.express as px + + spend_trend = px.line(daily_spend, x="log_date", y="spend", markers=True, + title="Daily spend (USD)", labels={"log_date": "Date (UTC)", "spend": "USD"}) + spend_trend.update_layout(template="plotly_white") + spend_trend +- cellId: 25ea0fa3-9a3c-58ed-949e-777ed64ce6dc + cellLabel: Spend by model + cellType: CODE + config: + source: |- + model_breakdown = px.bar(model_spend, x="model", y="spend", + title="Spend by model (USD)", labels={"model": "Model", "spend": "USD"}) + model_breakdown.update_layout(template="plotly_white") + model_breakdown +- cellId: da2a4acb-bc46-55f8-b6e4-bf71306912ca + cellLabel: Dashboard title + cellType: MARKDOWN + config: + source: |- + # LiteLLM spend + Last 30 UTC calendar days. Refresh the app to query uploaded spend logs. +appLayout: + visibleMetadataFields: + - LAST_RUN + tabs: + - name: Spend + rows: + - columns: + - start: 0 + end: 120 + elements: + - cellId: da2a4acb-bc46-55f8-b6e4-bf71306912ca + type: CELL + showSource: false + showLabel: false + - columns: + - start: 0 + end: 40 + elements: + - cellId: ab06dca2-ee5d-5b43-99f4-13dbbd6f0404 + type: CELL + showSource: false + showLabel: false + - start: 40 + end: 80 + elements: + - cellId: fc77975b-89e4-58de-ae86-846c534d2c25 + type: CELL + showSource: false + showLabel: false + - start: 80 + end: 120 + elements: + - cellId: 185f508e-ece0-5b72-9052-a27042ce4bdd + type: CELL + showSource: false + showLabel: false + - columns: + - start: 0 + end: 60 + elements: + - cellId: 856624aa-0a40-5c27-99e9-873c98b16d4d + type: CELL + showSource: false + showLabel: false + - start: 60 + end: 120 + elements: + - cellId: ae69da31-e28b-5ace-b622-4376380822cb + type: CELL + showSource: false + showLabel: false + - columns: + - start: 0 + end: 120 + elements: + - cellId: 55436d1d-3d4e-5068-b192-d540d488d9d0 + type: CELL + showSource: false + showLabel: false + - columns: + - start: 0 + end: 120 + elements: + - cellId: 25ea0fa3-9a3c-58ed-949e-777ed64ce6dc + type: CELL + showSource: false + showLabel: false + - columns: + - start: 0 + end: 120 + elements: + - cellId: 5199f238-552a-5d14-8368-e83ce7414369 + type: CELL + showSource: false + showLabel: false diff --git a/cookbook/logging_observability/hex_spend_table.sql b/cookbook/logging_observability/hex_spend_table.sql new file mode 100644 index 00000000000..f127163d5d2 --- /dev/null +++ b/cookbook/logging_observability/hex_spend_table.sql @@ -0,0 +1,40 @@ +-- Run once in Athena, in the database you will connect to Hex. +-- Replace YOUR_BUCKET and litellm-spend with HEX_S3_BUCKET_NAME and HEX_S3_PATH. +-- Keep query results in a separate prefix, outside this table's location. +-- Enable the LiteLLM callback with: litellm_settings: {callbacks: [hex]}. +-- Grant the proxy PutObject on the log prefix; grant Hex read-only access to +-- that prefix plus Athena/Glue query permissions and its query-results prefix. +-- Hex supports IAM role authentication; no Hex API key is needed by the proxy. +CREATE EXTERNAL TABLE IF NOT EXISTS litellm_hex_spend ( + id string, + start_time double, + end_time double, + model string, + model_group string, + custom_llm_provider string, + call_type string, + status string, + spend double, + prompt_tokens bigint, + completion_tokens bigint, + total_tokens bigint, + response_time double, + cache_hit boolean, + team_id string, + organization_id string, + user_id string, + key_alias string +) +PARTITIONED BY (log_date string) +ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' +STORED AS TEXTFILE +LOCATION 's3://YOUR_BUCKET/litellm-spend/' +TBLPROPERTIES ( + 'projection.enabled' = 'true', + 'projection.log_date.type' = 'date', + 'projection.log_date.range' = '2020-01-01,NOW', + 'projection.log_date.format' = 'yyyy-MM-dd', + 'projection.log_date.interval' = '1', + 'projection.log_date.interval.unit' = 'DAYS', + 'storage.location.template' = 's3://YOUR_BUCKET/litellm-spend/${log_date}/' +); diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..d7f1991ace9 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -138,6 +138,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "smtp_email", "deepeval", "s3_v2", + "hex", "clickhouse", "pointfive", "zerobus", diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index 38928f67f42..0b355b6599d 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -1,4 +1,31 @@ [ + { + "id": "hex", + "displayName": "Hex", + "logo": null, + "supports_key_team_logging": false, + "dynamic_params": { + "HEX_S3_BUCKET_NAME": { + "type": "text", + "ui_name": "S3 bucket name", + "description": "Bucket connected to Hex. AWS credentials come from the proxy's IAM role or standard AWS environment variables.", + "required": true + }, + "HEX_S3_PATH": { + "type": "text", + "ui_name": "S3 path prefix", + "description": "Object prefix for spend records. Defaults to litellm-spend.", + "required": false + }, + "HEX_S3_REGION_NAME": { + "type": "text", + "ui_name": "AWS region", + "description": "Bucket region. Defaults to AWS_REGION or us-east-1.", + "required": false + } + }, + "description": "Export individual spend records to S3 for Hex analysis and dashboards. Connect this bucket in Hex to query historical data." + }, { "id": "arize", "displayName": "Arize", diff --git a/litellm/integrations/hex.py b/litellm/integrations/hex.py new file mode 100644 index 00000000000..1c25d2c6c07 --- /dev/null +++ b/litellm/integrations/hex.py @@ -0,0 +1,80 @@ +"""Export request-level spend records to S3 for analysis in Hex.""" + +import os +from datetime import datetime +from typing import Final + +from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS +from litellm.integrations.s3_v2 import S3Logger +from litellm.types.integrations.s3_v2 import s3BatchLoggingElement +from litellm.types.utils import StandardLoggingPayload + + +class HexLogger(S3Logger): + """Write spend-only JSON records using the standard AWS credential chain. + + Enable with ``callbacks: ["hex"]`` and ``HEX_S3_BUCKET_NAME``. Connect the + bucket through Hex's Athena connection to query newly arriving records. + Prompts, responses, API keys, + arbitrary metadata, and error messages are never included in the export. + """ + + def __init__( + self, + bucket_name: str | None = None, + path: str | None = None, + region_name: str | None = None, + endpoint_url: str | None = None, + flush_interval: int = DEFAULT_S3_FLUSH_INTERVAL_SECONDS, + batch_size: int = DEFAULT_S3_BATCH_SIZE, + ) -> None: + resolved_bucket: Final = bucket_name or os.getenv("HEX_S3_BUCKET_NAME") + if not resolved_bucket or not resolved_bucket.strip(): + raise ValueError("Hex requires HEX_S3_BUCKET_NAME or bucket_name.") + if flush_interval <= 0 or batch_size <= 0: + raise ValueError("Hex flush_interval and batch_size must be positive.") + + super().__init__( # pyright: ignore[reportUnknownMemberType] # S3Logger has legacy untyped constructor options. + s3_bucket_name=resolved_bucket, + s3_path=path if path is not None else os.getenv("HEX_S3_PATH", "litellm-spend"), + s3_region_name=region_name or os.getenv("HEX_S3_REGION_NAME") or os.getenv("AWS_REGION") or "us-east-1", + s3_endpoint_url=endpoint_url or os.getenv("HEX_S3_ENDPOINT_URL"), + s3_flush_interval=flush_interval, + s3_batch_size=batch_size, + s3_batch_file_upload=True, + s3_partition_granularity="day", + # Keep the Hex destination independent of an existing S3 callback. + s3_callback_params_override={}, + ) + + def create_s3_batch_logging_element( + self, + start_time: datetime, + standard_logging_payload: StandardLoggingPayload | None, + ) -> s3BatchLoggingElement | None: + element: Final = super().create_s3_batch_logging_element(start_time, standard_logging_payload) + if element is None or standard_logging_payload is None: + return None + metadata: Final = standard_logging_payload["metadata"] + # An allowlist keeps arbitrary request content out of the analytics bucket. + element.payload = { + "id": standard_logging_payload["id"], + "start_time": standard_logging_payload["startTime"], + "end_time": standard_logging_payload["endTime"], + "model": standard_logging_payload["model"], + "model_group": standard_logging_payload["model_group"], + "custom_llm_provider": standard_logging_payload["custom_llm_provider"], + "call_type": standard_logging_payload["call_type"], + "status": standard_logging_payload["status"], + "spend": standard_logging_payload["response_cost"], + "prompt_tokens": standard_logging_payload["prompt_tokens"], + "completion_tokens": standard_logging_payload["completion_tokens"], + "total_tokens": standard_logging_payload["total_tokens"], + "response_time": standard_logging_payload["response_time"], + "cache_hit": standard_logging_payload["cache_hit"], + "team_id": metadata["user_api_key_team_id"], + "organization_id": metadata["user_api_key_org_id"], + "user_id": metadata["user_api_key_user_id"], + "key_alias": metadata["user_api_key_alias"], + } + return element diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index 74badc2c8c1..0455a41184a 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -29,6 +29,7 @@ from litellm.integrations.galileo import GalileoObserve from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger from litellm.integrations.gitlab import GitLabPromptManager +from litellm.integrations.hex import HexLogger from litellm.integrations.humanloop import HumanloopLogger from litellm.integrations.lago import LagoLogger from litellm.integrations.langfuse.langfuse_prompt_management import ( @@ -105,6 +106,7 @@ class CustomLoggerRegistry: "agentops": AgentOps, "deepeval": DeepEvalLogger, "s3_v2": S3Logger, + "hex": HexLogger, "pointfive": PointFiveLogger, "zerobus": ZerobusLogger, "aws_sqs": SQSLogger, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ce57a9bd4a5..1ae281cc563 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -74,6 +74,7 @@ from litellm.integrations.arize.arize import ArizeLogger from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.deepeval.deepeval import DeepEvalLogger +from litellm.integrations.hex import HexLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger @@ -5791,12 +5792,19 @@ def build_custom_logger_compatible_class( return _gcs_bucket_logger elif logging_integration == "s3_v2": for callback in _in_memory_loggers: - if isinstance(callback, S3V2Logger): + if type(callback) is S3V2Logger: return callback _s3_v2_logger: Final = S3V2Logger() _in_memory_loggers.append(_s3_v2_logger) return _s3_v2_logger + elif logging_integration == "hex": + for callback in _in_memory_loggers: + if isinstance(callback, HexLogger): + return callback + hex_logger: Final = HexLogger() + _in_memory_loggers.append(hex_logger) + return hex_logger elif logging_integration == "clickhouse": for callback in _in_memory_loggers: if isinstance(callback, ClickHouseSpendLogger): @@ -6613,7 +6621,11 @@ def get_custom_logger_compatible_class( return callback elif logging_integration == "s3_v2": for callback in _in_memory_loggers: - if isinstance(callback, S3V2Logger): + if type(callback) is S3V2Logger: + return callback + elif logging_integration == "hex": + for callback in _in_memory_loggers: + if isinstance(callback, HexLogger): return callback elif logging_integration == "clickhouse": for callback in _in_memory_loggers: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9a80761b477..2dd8bf2d74f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7200,7 +7200,7 @@ class ProxyConfig: ) reset_audit_log_callback_cache() - _in_memory_loggers[:] = [cb for cb in _in_memory_loggers if not isinstance(cb, S3V2Logger)] + _in_memory_loggers[:] = [cb for cb in _in_memory_loggers if type(cb) is not S3V2Logger] if redis_usage_cache is None: env_coordination_redis_cache: Final = await self._init_coordination_redis_env_fallback( diff --git a/tests/unit/integrations/test_hex.py b/tests/unit/integrations/test_hex.py new file mode 100644 index 00000000000..358ec9282e8 --- /dev/null +++ b/tests/unit/integrations/test_hex.py @@ -0,0 +1,190 @@ +import json +from datetime import datetime, timezone +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.integrations.hex import HexLogger +from litellm.integrations.s3_v2 import S3Logger +from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry +from litellm.litellm_core_utils.litellm_logging import ( + build_custom_logger_compatible_class, + create_dummy_standard_logging_payload, + get_custom_logger_compatible_class, +) +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy.management_endpoints.callback_management_endpoints import get_callback_configs +from litellm.types.utils import StandardLoggingPayload + +START: Final = datetime(2026, 10, 30, 12, 0, tzinfo=timezone.utc) + + +@pytest.fixture +def spend_record() -> StandardLoggingPayload: + payload: Final = create_dummy_standard_logging_payload() + payload.update( + id="request-1", + startTime=START.timestamp(), + endTime=START.timestamp() + 1, + response_cost=0.125, + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + response_time=1.0, + messages=[{"role": "user", "content": "private prompt"}], + response="private response", + error_str="private provider error", + ) + payload["metadata"]["requester_metadata"] = {"private": "must not be exported"} + return payload + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ["success", "failure"]) +async def test_hex_exports_spend_rows_without_request_content( + spend_record: StandardLoggingPayload, status: str +) -> None: + logger: Final = HexLogger(bucket_name="hex-analytics") + if status == "failure": + spend_record["status"] = "failure" + await logger.async_log_failure_event({"standard_logging_object": spend_record}, None, START, START) + else: + await logger.async_log_success_event({"standard_logging_object": spend_record}, None, START, START) + + assert len(logger.log_queue) == 1 + element: Final = logger.log_queue[0] + assert element.payload == { + "id": "request-1", + "start_time": START.timestamp(), + "end_time": START.timestamp() + 1, + "model": "gpt-3.5-turbo", + "model_group": "openai-gpt", + "custom_llm_provider": "openai", + "call_type": "completion", + "status": status, + "spend": 0.125, + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + "response_time": 1.0, + "cache_hit": False, + "team_id": "test_team", + "organization_id": None, + "user_id": "test_user", + "key_alias": "test_alias", + } + assert element.s3_object_key == "litellm-spend/2026-10-30/time-12-00-00-000000_request-1.json" + assert "private" not in logger._prepare_put(element).json_string + assert spend_record["messages"] == [{"role": "user", "content": "private prompt"}] + + +@pytest.mark.asyncio +async def test_hex_skips_events_without_a_spend_payload() -> None: + logger: Final = HexLogger(bucket_name="hex-analytics") + await logger.async_log_success_event({}, None, START, START) + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_hex_retains_a_rejected_spend_record_until_the_next_flush( + monkeypatch: pytest.MonkeyPatch, spend_record: StandardLoggingPayload +) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "synthetic-access-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "synthetic-signing-key") + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(400 if len(requests) == 1 else 200) + + logger: Final = HexLogger(bucket_name="hex-analytics", endpoint_url="https://s3.example.test") + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + logger.async_httpx_client = handler + try: + await logger.async_log_success_event({"standard_logging_object": spend_record}, None, START, START) + await logger.flush_queue() + assert len(logger.log_queue) == 1 + await logger.flush_queue() + assert logger.log_queue == [] + assert len(requests) == 2 + assert requests[0].content == requests[1].content + assert str(requests[1].url).startswith("https://s3.example.test/hex-analytics/litellm-spend/2026-10-30/batch_") + assert str(requests[1].url).endswith(".jsonl") + assert json.loads(requests[1].content)["spend"] == 0.125 + assert b"private" not in requests[1].content + finally: + await handler.client.aclose() + + +@pytest.mark.asyncio +async def test_hex_uploads_multiple_requests_as_individual_rows_in_one_object( + monkeypatch: pytest.MonkeyPatch, spend_record: StandardLoggingPayload +) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "synthetic-access-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "synthetic-signing-key") + monkeypatch.setenv("S3_PARTITION_GRANULARITY", "hour") + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200) + + logger: Final = HexLogger(bucket_name="hex-analytics", endpoint_url="https://s3.example.test") + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + logger.async_httpx_client = handler + try: + for request_id in ("request-1", "request-2"): + spend_record["id"] = request_id + await logger.async_log_success_event({"standard_logging_object": spend_record}, None, START, START) + await logger.flush_queue() + assert len(requests) == 1 + rows: Final = [json.loads(line) for line in requests[0].content.splitlines()] + assert [row["id"] for row in rows] == ["request-1", "request-2"] + assert sum(row["spend"] for row in rows) == 0.25 + assert "/2026-10-30/batch_" in str(requests[0].url) + assert requests[0].headers["content-type"] == "application/x-ndjson" + assert b"private" not in requests[0].content + finally: + await handler.client.aclose() + + +def test_hex_requires_a_bucket(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("HEX_S3_BUCKET_NAME", raising=False) + with pytest.raises(ValueError, match="HEX_S3_BUCKET_NAME"): + HexLogger() + + +@pytest.mark.asyncio +async def test_hex_uses_its_own_destination_and_reuses_the_registered_instance(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("HEX_S3_BUCKET_NAME", "hex-analytics") + monkeypatch.setenv("HEX_S3_PATH", "analytics") + monkeypatch.setenv("HEX_S3_REGION_NAME", "eu-west-1") + monkeypatch.setattr(litellm, "s3_callback_params", {"s3_bucket_name": "other-bucket", "s3_path": "other-path"}) + + hex_logger: Final = build_custom_logger_compatible_class("hex", None, None) + s3_logger: Final = build_custom_logger_compatible_class("s3_v2", None, None) + assert isinstance(hex_logger, HexLogger) + assert type(s3_logger) is S3Logger + assert (hex_logger.s3_bucket_name, hex_logger.s3_path, hex_logger.s3_region_name) == ( + "hex-analytics", + "analytics", + "eu-west-1", + ) + assert s3_logger.s3_bucket_name == "other-bucket" + assert build_custom_logger_compatible_class("hex", None, None) is hex_logger + assert get_custom_logger_compatible_class("hex") is hex_logger + assert get_custom_logger_compatible_class("s3_v2") is s3_logger + assert CustomLoggerRegistry.get_callback_str_from_class_type(HexLogger) == "hex" + + +@pytest.mark.asyncio +async def test_hex_settings_list_the_required_bucket() -> None: + configs: Final = await get_callback_configs() + assert isinstance(configs, list) + config: Final = next(item for item in configs if isinstance(item, dict) and item.get("id") == "hex") + assert config["displayName"] == "Hex" + assert config["supports_key_team_logging"] is False + assert config["dynamic_params"]["HEX_S3_BUCKET_NAME"]["required"] is True + assert "API_KEY" not in json.dumps(config) diff --git a/tests/unit/litellm_core_utils/test_custom_logger_registry.py b/tests/unit/litellm_core_utils/test_custom_logger_registry.py index 5fc21d75e3e..38cac6c8b2d 100644 --- a/tests/unit/litellm_core_utils/test_custom_logger_registry.py +++ b/tests/unit/litellm_core_utils/test_custom_logger_registry.py @@ -26,6 +26,7 @@ for collector in collectors: expected_env_vars = { + "HEX_S3_BUCKET_NAME": "hex-analytics", "LAGO_API_KEY": "api_key", "LAGO_API_BASE": "mock_base", "LAGO_API_EVENT_CODE": "mock_event_code",