diff --git a/litellm/integrations/SlackAlerting/utils.py b/litellm/integrations/SlackAlerting/utils.py index 4424bedba81..d137e6ebfd8 100644 --- a/litellm/integrations/SlackAlerting/utils.py +++ b/litellm/integrations/SlackAlerting/utils.py @@ -84,7 +84,7 @@ async def _add_langfuse_trace_id_to_alert( ######################################################### langfuse_object = litellm_logging_obj._get_callback_object(service_name="langfuse") if langfuse_object is not None: - base_url = langfuse_object.Langfuse.base_url + base_url = langfuse_object.langfuse_host return f"{base_url}/trace/{trace_id}" return None diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index c50e1e3151f..cdeff86b4bd 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -751,7 +751,7 @@ class LangFuseLogger: generation_client.end(end_time=end_time_ns) trace.end(end_time=end_time_ns) - return trace.trace_id, generation_client.id + return trace_context["trace_id"], generation_client.id except Exception: verbose_logger.error(f"Langfuse Layer Error - {traceback.format_exc()}") return None, None diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index 697c3837602..1bf5cd52c41 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -930,6 +930,7 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): - Unit test for `_get_trace_id` function in Logging obj """ from litellm.litellm_core_utils.litellm_logging import Logging + from langfuse import Langfuse litellm.success_callback = ["langfuse"] litellm_call_id = "my-unique-call-id" @@ -961,24 +962,23 @@ def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): time.sleep(3) assert litellm_logging_obj._get_trace_id(service_name="langfuse") is not None - ## if existing_trace_id exists - if langfuse_existing_trace_id is not None: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == langfuse_existing_trace_id - ) - ## if trace_id exists - elif langfuse_trace_id is not None: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == langfuse_trace_id - ) - ## if no trace_id or existing_trace_id is provided, use litellm_trace_id - else: - assert ( - litellm_logging_obj._get_trace_id(service_name="langfuse") - == litellm_logging_obj.litellm_trace_id - ) + source_trace_id = ( + langfuse_existing_trace_id + or langfuse_trace_id + or litellm_logging_obj.litellm_trace_id + ) + normalized_trace_id = source_trace_id.lower().replace("-", "") + expected_trace_id = ( + normalized_trace_id + if len(normalized_trace_id) == 32 + and normalized_trace_id != "0" * 32 + and all(character in "0123456789abcdef" for character in normalized_trace_id) + else Langfuse.create_trace_id(seed=source_trace_id) + ) + assert ( + litellm_logging_obj._get_trace_id(service_name="langfuse") + == expected_trace_id + ) def test_convert_model_response_object(): diff --git a/tests/local_testing/test_alangfuse.py b/tests/local_testing/test_alangfuse.py index 7c2ec7e9f64..195119b08ce 100644 --- a/tests/local_testing/test_alangfuse.py +++ b/tests/local_testing/test_alangfuse.py @@ -231,12 +231,16 @@ async def test_langfuse_logging_without_request_response(stream, langfuse_client langfuse_client.flush() await asyncio.sleep(5) - # get trace with _unique_trace_name - trace = langfuse_client.get_generations(trace_id=_unique_trace_name) + trace_id = langfuse_client.create_trace_id(seed=_unique_trace_name) + observations = langfuse_client.api.observations.get_many( + trace_id=trace_id, + type="GENERATION", + parse_io_as_json=True, + ) - print("trace_from_langfuse", trace) + print("observations_from_langfuse", observations) - _trace_data = trace.data + _trace_data = observations.data if ( len(_trace_data) == 0 @@ -292,11 +296,16 @@ async def test_langfuse_logging_audio_transcriptions(langfuse_client): langfuse_client.flush() await asyncio.sleep(20) - # get trace with _unique_trace_name print("lookiing up trace", _unique_trace_name) - trace = langfuse_client.get_trace(id=_unique_trace_name) + trace_id = langfuse_client.create_trace_id(seed=_unique_trace_name) generations = list( - reversed(langfuse_client.get_generations(trace_id=_unique_trace_name).data) + reversed( + langfuse_client.api.observations.get_many( + trace_id=trace_id, + type="GENERATION", + parse_io_as_json=True, + ).data + ) ) print("generations for given trace=", generations) @@ -338,11 +347,17 @@ async def test_langfuse_masked_input_output(langfuse_client): langfuse_client.flush() await asyncio.sleep(30) - # get trace with _unique_trace_name - trace = langfuse_client.get_trace(id=_unique_trace_name) + trace_id = langfuse_client.create_trace_id(seed=_unique_trace_name) + trace = langfuse_client.api.trace.get(trace_id) print("trace_from_langfuse", trace) generations = list( - reversed(langfuse_client.get_generations(trace_id=_unique_trace_name).data) + reversed( + langfuse_client.api.observations.get_many( + trace_id=trace_id, + type="GENERATION", + parse_io_as_json=True, + ).data + ) ) assert expected_input in str(trace.input) @@ -443,17 +458,24 @@ async def test_aaalangfuse_logging_metadata(langfuse_client): # Tests the metadata filtering and the override of the output to be the last generation for trace_id, generation_ids in trace_identifiers.items(): + resolved_trace_id = langfuse_client.create_trace_id(seed=trace_id) try: - trace = langfuse_client.get_trace(id=trace_id) + trace = langfuse_client.api.trace.get(resolved_trace_id) except Exception as e: if "not found within authorized project" in str(e): print(f"Trace {trace_id} not found") continue - assert trace.id == trace_id + assert trace.id == resolved_trace_id assert trace.session_id == session_id assert trace.metadata != trace_metadata generations = list( - reversed(langfuse_client.get_generations(trace_id=trace_id).data) + reversed( + langfuse_client.api.observations.get_many( + trace_id=resolved_trace_id, + type="GENERATION", + parse_io_as_json=True, + ).data + ) ) assert len(generations) == len(generation_ids) assert ( @@ -470,7 +492,7 @@ async def test_aaalangfuse_logging_metadata(langfuse_client): print("trace_from_langfuse", trace) for generation_id, generation in zip(generation_ids, generations): assert generation.id == generation_id - assert generation.trace_id == trace_id + assert generation.trace_id == resolved_trace_id print( "common keys in trace", set(generation.metadata.keys()).intersection( diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json deleted file mode 100644 index e252e8a128f..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "batch": [ - { - "id": "7e00e081-468b-4fe9-a409-eb12ac7d3d2d", - "type": "trace-create", - "body": { - "id": "litellm-test-793c217f-9417-4e77-84a7-8dcc16e5b72b", - "timestamp": "2025-01-16T19:28:55.124873Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-16T19:28:55.125002Z" - }, - { - "id": "b9ec2c0f-18df-46c7-9e90-624c60bf78ee", - "type": "generation-create", - "body": { - "name": "litellm-acompletion", - "startTime": "2025-01-16T11:28:54.796360-08:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 3.5e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 3.5e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-11-28-54-796360_chatcmpl-521e530f-5e29-4d0a-8d1a-58fca0a847c2", - "endTime": "2025-01-16T11:28:55.124353-08:00", - "completionStartTime": "2025-01-16T11:28:55.124353-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - }, - "traceId": "litellm-test-6a51ae70-a4e7-499e-afcd-dce2a3b31850" - }, - "timestamp": "2025-01-16T19:28:55.125258Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-03734ab3-8790-4c09-b5fb-8c3b663413b6" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json deleted file mode 100644 index dd49d9751f1..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "batch": [ - { - "id": "3c9b544f-ef3f-449e-8ec1-763acbb56bec", - "type": "trace-create", - "body": { - "id": "litellm-test-c4c1c850-e8c9-4b16-b5a4-bff2bf9fa4f6", - "timestamp": "2025-05-26T21:13:16.796768Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "tags": [] - }, - "timestamp": "2025-05-26T21:13:16.796875Z" - }, - { - "id": "90e6bc70-05d9-4444-8b87-4523a9a54c17", - "type": "generation-create", - "body": { - "traceId": "litellm-test-c4c1c850-e8c9-4b16-b5a4-bff2bf9fa4f6", - "name": "litellm-acompletion", - "startTime": "2025-05-26T14:13:16.469836-07:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": null, - "response_cost": 6e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - "usage_object": null - }, - "litellm_response_cost": 6e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "level": "DEFAULT", - "id": "time-14-13-16-469836_chatcmpl-3803a9e9-aa68-4493-94d9-247f354830d6", - "endTime": "2025-05-26T14:13:16.795438-07:00", - "completionStartTime": "2025-05-26T14:13:16.795438-07:00", - "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - "modelParameters": { - "aws_region": "us-east-1" - }, - "usage": { - "input": 10, - "output": 10, - "unit": "TOKENS", - "totalCost": 6e-05 - }, - "usageDetails": { - "input": 10, - "output": 10, - "total": 20, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-05-26T21:13:16.797156Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-3bfc4db9-217f-48e9-92e0-142566e3c204" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json deleted file mode 100644 index 15794de7a07..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "batch": [ - { - "id": "9ee9100b-c4aa-4e40-a10d-bc189f8b4242", - "type": "trace-create", - "body": { - "id": "litellm-test-c414db10-dd68-406e-9d9e-03839bc2f346", - "timestamp": "2025-01-22T17:27:51.702596Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:27:51.702716Z" - }, - { - "id": "f8d20489-ed58-429f-b609-87380e223746", - "type": "generation-create", - "body": { - "traceId": "litellm-test-c414db10-dd68-406e-9d9e-03839bc2f346", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:27:51.150898-08:00", - "metadata": { - "string_value": "hello", - "int_value": 42, - "float_value": 3.14, - "bool_value": true, - "nested_dict": { - "key1": "value1", - "key2": { - "inner_key": "inner_value" - } - }, - "list_value": [ - 1, - 2, - 3 - ], - "set_value": [ - 1, - 2, - 3 - ], - "complex_list": [ - { - "dict_in_list": "value" - }, - "simple_string", - [ - 1, - 2, - 3 - ] - ], - "user": { - "name": "John", - "age": 30, - "tags": [ - "customer", - "active" - ] - }, - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-27-51-150898_chatcmpl-b783291c-dc76-4660-bfef-b79be9d54e57", - "endTime": "2025-01-22T09:27:51.702048-08:00", - "completionStartTime": "2025-01-22T09:27:51.702048-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:27:51.703046Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json deleted file mode 100644 index 8d5d08894ef..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "batch": [ - { - "id": "872a0a1c-4328-431b-80b6-fd55a8a44477", - "type": "trace-create", - "body": { - "id": "litellm-test-533ffb2d-a0a3-45b5-911c-7940466cdc8e", - "timestamp": "2025-01-22T17:19:11.234960Z", - "name": "test_trace_name", - "userId": "test_user_id", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "sessionId": "test_session_id", - "version": "test_trace_version", - "metadata": { - "test_key": "test_value" - }, - "tags": [ - "test_tag", - "test_tag_2" - ] - }, - "timestamp": "2025-01-22T17:19:11.235169Z" - }, - { - "id": "18d6f044-e522-4376-96e0-7eec765677ed", - "type": "generation-create", - "body": { - "traceId": "litellm-test-533ffb2d-a0a3-45b5-911c-7940466cdc8e", - "name": "test_generation_name", - "startTime": "2025-01-22T09:19:10.957072-08:00", - "metadata": { - "tags": [ - "test_tag", - "test_tag_2" - ], - "parent_observation_id": "test_parent_observation_id", - "version": "test_version", - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 3.5e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 3.5e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "parentObservationId": "test_parent_observation_id", - "version": "test_version", - "id": "time-09-19-10-957072_chatcmpl-4da65aba-32e4-400d-aaa2-6bfe096d8141", - "endTime": "2025-01-22T09:19:11.234200-08:00", - "completionStartTime": "2025-01-22T09:19:11.234200-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:19:11.235541Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json deleted file mode 100644 index ff8419ee392..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "batch": [ - { - "id": "1f1d7517-4602-4c59-a322-7fc0306f1b7a", - "type": "trace-create", - "body": { - "id": "litellm-test-dbadfdfc-f4e7-4f05-8992-984c37359166", - "timestamp": "2025-02-07T00:23:27.669634Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "tags": [] - }, - "timestamp": "2025-02-07T00:23:27.669809Z" - }, - { - "id": "fbe610b6-f500-4c7d-8e34-d40a0e8c487b", - "type": "generation-create", - "body": { - "traceId": "litellm-test-dbadfdfc-f4e7-4f05-8992-984c37359166", - "name": "litellm-acompletion", - "startTime": "2025-02-06T16:23:27.220129-08:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 3.5e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 3.5e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "level": "DEFAULT", - "id": "time-16-23-27-220129_chatcmpl-565360d7-965f-4533-9c09-db789af77a7d", - "endTime": "2025-02-06T16:23:27.644253-08:00", - "completionStartTime": "2025-02-06T16:23:27.644253-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 10, - "unit": "TOKENS", - "totalCost": 1.9999999999999998e-05 - }, - "usageDetails": { - "input": 10, - "output": 10, - "total": 20, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-02-07T00:23:27.670175Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json deleted file mode 100644 index df99b11d26b..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "batch": [ - { - "id": "45eb9b25-605c-4c4a-b2b3-8241e079cd31", - "type": "trace-create", - "body": { - "id": "litellm-test-32702f3d-8a1c-4912-a3d6-286e59a9c568", - "timestamp": "2025-05-24T17:01:19.408179Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "tags": [] - }, - "timestamp": "2025-05-24T17:01:19.408284Z" - }, - { - "id": "9f5e9b7d-0cea-4776-b4b9-5c2e8f4bad3c", - "type": "generation-create", - "body": { - "traceId": "litellm-test-32702f3d-8a1c-4912-a3d6-286e59a9c568", - "name": "litellm-acompletion", - "startTime": "2025-05-24T10:01:19.142356-07:00", - "metadata": { - "model_group": "gpt-3.5-turbo", - "model_group_size": 1, - "deployment": "gpt-3.5-turbo", - "model_info": { - "id": "0f1cd8f9e6a22e499303d479486395563ea04decade83fe7334dc2f079a857c2", - "db_model": false - }, - "api_base": null, - "hidden_params": { - "model_id": "0f1cd8f9e6a22e499303d479486395563ea04decade83fe7334dc2f079a857c2", - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 3.5e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 3.5e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "level": "DEFAULT", - "id": "time-10-01-19-142356_chatcmpl-16b215b7-e51e-47b0-8fe5-9dd6f226fda1", - "endTime": "2025-05-24T10:01:19.406531-07:00", - "completionStartTime": "2025-05-24T10:01:19.406531-07:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "stream": false, - "max_retries": 0, - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 10, - "unit": "TOKENS", - "totalCost": 1.9999999999999998e-05 - }, - "usageDetails": { - "input": 10, - "output": 10, - "total": 20, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-05-24T17:01:19.408586Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-3bfc4db9-217f-48e9-92e0-142566e3c204" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json deleted file mode 100644 index fd3d3194a5b..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "batch": [ - { - "id": "42be960a-5dde-47df-9cbc-1fdd0fdcaa7d", - "type": "trace-create", - "body": { - "id": "litellm-test-f3ab679b-1e1d-43fd-9a9a-f11287aeb339", - "timestamp": "2025-01-22T15:31:28.963419Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [ - "test_tag", - "test_tag_2" - ] - }, - "timestamp": "2025-01-22T15:31:28.963706Z" - }, - { - "id": "5486df5a-3776-4adf-abd0-bd22e51f7fb4", - "type": "generation-create", - "body": { - "traceId": "litellm-test-f3ab679b-1e1d-43fd-9a9a-f11287aeb339", - "name": "litellm-acompletion", - "startTime": "2025-01-22T07:31:28.960749-08:00", - "metadata": { - "tags": [ - "test_tag", - "test_tag_2" - ], - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-07-31-28-960749_chatcmpl-f06338f0-8c49-45d8-be35-2854a89723c1", - "endTime": "2025-01-22T07:31:28.962389-08:00", - "completionStartTime": "2025-01-22T07:31:28.962389-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T15:31:28.964179Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json deleted file mode 100644 index af15f351189..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "batch": [ - { - "id": "06b8fa9f-151b-4e74-9fbf-8af5222a7f40", - "type": "trace-create", - "body": { - "id": "litellm-test-54368a51-a382-493c-b0a8-3f1af23e18c4", - "timestamp": "2025-01-22T16:38:26.016582Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [ - "test_tag_stream", - "test_tag_2_stream" - ] - }, - "timestamp": "2025-01-22T16:38:26.016828Z" - }, - { - "id": "4ca1fd78-53e3-41b5-95d9-417b09e3f0eb", - "type": "generation-create", - "body": { - "traceId": "litellm-test-54368a51-a382-493c-b0a8-3f1af23e18c4", - "name": "litellm-acompletion", - "startTime": "2025-01-22T08:38:25.665692-08:00", - "metadata": { - "tags": [ - "test_tag_stream", - "test_tag_2_stream" - ], - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-08-38-25-665692_chatcmpl-8b67ffb8-4326-4e1b-bf4a-f70930c11c00", - "endTime": "2025-01-22T08:38:26.015666-08:00", - "completionStartTime": "2025-01-22T08:38:26.015666-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T16:38:26.017252Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json deleted file mode 100644 index b6c11f96953..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "batch": [ - { - "id": "7d33d536-2730-4815-8957-80866c09c053", - "type": "trace-create", - "body": { - "id": "litellm-test-72861437-ff5b-4c48-89c0-a143534d9e7a", - "timestamp": "2025-05-26T21:15:40.610459Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "tags": [] - }, - "timestamp": "2025-05-26T21:15:40.610603Z" - }, - { - "id": "ebb5079c-7726-4adb-9616-e1862735e1d8", - "type": "generation-create", - "body": { - "traceId": "litellm-test-72861437-ff5b-4c48-89c0-a143534d9e7a", - "name": "litellm-acompletion", - "startTime": "2025-05-26T14:15:40.349639-07:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": null, - "response_cost": 7.5e-06, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "vertex_ai/gemini-2.0-flash-001", - "usage_object": null - }, - "litellm_response_cost": 7.5e-06, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "level": "DEFAULT", - "id": "time-14-15-40-349639_chatcmpl-59a988d0-7ef1-4dc4-bc18-d2e78961817f", - "endTime": "2025-05-26T14:15:40.607266-07:00", - "completionStartTime": "2025-05-26T14:15:40.607266-07:00", - "model": "gemini-2.0-flash-001", - "modelParameters": {}, - "usage": { - "input": 10, - "output": 10, - "unit": "TOKENS", - "totalCost": 7.5e-06 - }, - "usageDetails": { - "input": 10, - "output": 10, - "total": 20, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-05-26T21:15:40.610953Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-3bfc4db9-217f-48e9-92e0-142566e3c204" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json deleted file mode 100644 index 82a115a0899..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "batch": [ - { - "id": "ddf567e5-a1b5-4e38-8a7c-f48bc847f721", - "type": "trace-create", - "body": { - "id": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", - "timestamp": "2025-01-22T17:59:39.367430Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:59:39.367707Z" - }, - { - "id": "d3eb2c9e-e123-419d-b27b-c8283a505ae8", - "type": "generation-create", - "body": { - "traceId": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:59:39.362554-08:00", - "metadata": { - "int": 42, - "str": "hello", - "list": [ - 1, - 2, - 3 - ], - "set": [ - 4, - 5 - ], - "dict": { - "nested": "value" - }, - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-59-39-362554_chatcmpl-d20ba1d9-cda6-4773-822e-921ebcd426a0", - "endTime": "2025-01-22T09:59:39.365756-08:00", - "completionStartTime": "2025-01-22T09:59:39.365756-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:59:39.368310Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json deleted file mode 100644 index 33e6b01bee3..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "batch": [ - { - "id": "ea3d694a-ce6b-417e-86e3-23ac17c6f6c6", - "type": "trace-create", - "body": { - "id": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", - "timestamp": "2025-01-22T18:06:50.959206Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T18:06:50.959409Z" - }, - { - "id": "5fe03133-5798-4f87-8eec-ae0264f1eccc", - "type": "generation-create", - "body": { - "traceId": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", - "name": "litellm-acompletion", - "startTime": "2025-01-22T10:06:50.957097-08:00", - "metadata": { - "list": [ - "list", - "not", - "a", - "dict" - ], - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-10-06-50-957097_chatcmpl-62d4ad7c-291b-4fc7-a8a4-3ed0fc3912a5", - "endTime": "2025-01-22T10:06:50.958374-08:00", - "completionStartTime": "2025-01-22T10:06:50.958374-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T18:06:50.959850Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/embedding_with_vllm.json b/tests/logging_callback_tests/langfuse_expected_request_body/embedding_with_vllm.json deleted file mode 100644 index f8621adadbb..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/embedding_with_vllm.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "model": "BAAI/bge-small-en-v1.5", - "input": ["Hello from litellm!"] -} diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json deleted file mode 100644 index f4040f1f8fc..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "batch": [ - { - "id": "28d0c943-284b-4151-bf0d-8acf0f449865", - "type": "trace-create", - "body": { - "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", - "timestamp": "2025-01-22T17:59:32.888622Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:59:32.888940Z" - }, - { - "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", - "type": "generation-create", - "body": { - "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:59:32.878577-08:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", - "endTime": "2025-01-22T09:59:32.880691-08:00", - "completionStartTime": "2025-01-22T09:59:32.880691-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:59:32.889548Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json deleted file mode 100644 index 77ca252c86d..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "batch": [ - { - "id": "88b1898a-cc5d-4e8e-93bc-3e71300c5e8d", - "type": "trace-create", - "body": { - "id": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", - "timestamp": "2025-01-22T17:59:36.162545Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:59:36.162702Z" - }, - { - "id": "96bb77a6-a350-431b-bfd8-425491259728", - "type": "generation-create", - "body": { - "traceId": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:59:36.161090-08:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-59-36-161090_chatcmpl-1ee988c9-9133-4655-bbe4-b97ffb6e3dc9", - "endTime": "2025-01-22T09:59:36.161959-08:00", - "completionStartTime": "2025-01-22T09:59:36.161959-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:59:36.162997Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json deleted file mode 100644 index f4040f1f8fc..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "batch": [ - { - "id": "28d0c943-284b-4151-bf0d-8acf0f449865", - "type": "trace-create", - "body": { - "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", - "timestamp": "2025-01-22T17:59:32.888622Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:59:32.888940Z" - }, - { - "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", - "type": "generation-create", - "body": { - "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:59:32.878577-08:00", - "metadata": { - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", - "endTime": "2025-01-22T09:59:32.880691-08:00", - "completionStartTime": "2025-01-22T09:59:32.880691-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:59:32.889548Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json deleted file mode 100644 index f4a1bb9dcea..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "batch": [ - { - "id": "44f179be-e3b9-486f-986f-030fc50614f0", - "type": "trace-create", - "body": { - "id": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", - "timestamp": "2025-01-22T17:55:28.854927Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:55:28.855187Z" - }, - { - "id": "2175ee64-58a3-41ab-96df-405b76695f5f", - "type": "generation-create", - "body": { - "traceId": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:55:28.852503-08:00", - "metadata": { - "a": { - "nested_a": 1 - }, - "b": { - "nested_b": 2 - }, - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-55-28-852503_chatcmpl-131cf0da-a47b-4cd1-850b-50fa077362ac", - "endTime": "2025-01-22T09:55:28.853979-08:00", - "completionStartTime": "2025-01-22T09:55:28.853979-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:55:28.855732Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json deleted file mode 100644 index d895378e2c6..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "batch": [ - { - "id": "02c74119-76b7-4f79-91cb-c55f1495c100", - "type": "trace-create", - "body": { - "id": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", - "timestamp": "2025-01-22T17:53:53.754012Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:53:53.754178Z" - }, - { - "id": "097968e0-52e9-46b5-9e8e-e6e08dd00e72", - "type": "generation-create", - "body": { - "traceId": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:53:53.752422-08:00", - "metadata": { - "a": { - "nested_a": 1 - }, - "b": { - "nested_b": 2 - }, - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-53-53-752422_chatcmpl-e99bc1d3-a393-493f-8afe-4507c0acff15", - "endTime": "2025-01-22T09:53:53.753431-08:00", - "completionStartTime": "2025-01-22T09:53:53.753431-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:53:53.754511Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json deleted file mode 100644 index 87eba33cfff..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "batch": [ - { - "id": "1a55383a-e6fa-41f9-81fe-e7aa58c55f40", - "type": "trace-create", - "body": { - "id": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", - "timestamp": "2025-01-22T17:56:35.477276Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:56:35.477571Z" - }, - { - "id": "13ba66e8-f72b-4f57-a6cc-57c0be2829b1", - "type": "generation-create", - "body": { - "traceId": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:56:35.474752-08:00", - "metadata": { - "a": [ - 1, - 2, - 3 - ], - "b": [ - 4, - 5, - 6 - ], - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-56-35-474752_chatcmpl-9b152610-3d1e-4731-a84e-d0341ea69a0f", - "endTime": "2025-01-22T09:56:35.476236-08:00", - "completionStartTime": "2025-01-22T09:56:35.476236-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:56:35.478171Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json deleted file mode 100644 index dd3bb4a301f..00000000000 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "batch": [ - { - "id": "7fb1f295-a7af-47af-afbd-e2f2d08280aa", - "type": "trace-create", - "body": { - "id": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", - "timestamp": "2025-01-22T17:56:38.786515Z", - "name": "litellm-acompletion", - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "tags": [] - }, - "timestamp": "2025-01-22T17:56:38.786742Z" - }, - { - "id": "412870bc-fc50-4426-a0dc-9e8b016e14bb", - "type": "generation-create", - "body": { - "traceId": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", - "name": "litellm-acompletion", - "startTime": "2025-01-22T09:56:38.784548-08:00", - "metadata": { - "a": [ - 1, - 2 - ], - "b": [ - 3, - 4 - ], - "c": { - "d": [ - 5, - 6 - ] - }, - "hidden_params": { - "model_id": null, - "cache_key": null, - "api_base": "https://api.openai.com", - "response_cost": 5.4999999999999995e-05, - "additional_headers": {}, - "litellm_overhead_time_ms": null, - "batch_models": null, - "litellm_model_name": "gpt-3.5-turbo", - "usage_object": null - }, - "litellm_response_cost": 5.4999999999999995e-05, - "cache_hit": false, - "requester_metadata": {} - }, - "input": { - "messages": [ - { - "role": "user", - "content": "Hello!" - } - ] - }, - "output": { - "content": "Hello! How can I assist you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null, - "provider_specific_fields": null - }, - "level": "DEFAULT", - "id": "time-09-56-38-784548_chatcmpl-438c8727-86b3-44d9-9b46-42330922cf50", - "endTime": "2025-01-22T09:56:38.785762-08:00", - "completionStartTime": "2025-01-22T09:56:38.785762-08:00", - "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, - "usage": { - "input": 10, - "output": 20, - "unit": "TOKENS", - "totalCost": 3.5e-05 - }, - "usageDetails": { - "input": 10, - "output": 20, - "total": 30, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } - }, - "timestamp": "2025-01-22T17:56:38.787196Z" - } - ], - "metadata": { - "batch_size": 2, - "sdk_integration": "litellm", - "sdk_name": "python", - "sdk_version": "2.44.1", - "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" - } -} \ No newline at end of file diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py deleted file mode 100644 index bc64e30738f..00000000000 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ /dev/null @@ -1,595 +0,0 @@ -import asyncio -import copy -import json -import logging -import os -import sys -import threading -from typing import Any, Optional -from unittest.mock import AsyncMock, MagicMock, patch - -import httpx - -logging.basicConfig(level=logging.DEBUG) -sys.path.insert(0, os.path.abspath("../..")) - -import litellm -from litellm import completion -from litellm.caching import InMemoryCache -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - -litellm.num_retries = 3 -litellm.success_callback = ["langfuse"] -os.environ["LANGFUSE_DEBUG"] = "True" -import time - -import pytest -import pytest_asyncio - - -def assert_langfuse_request_matches_expected( - actual_request_body: dict, - expected_file_name: str, - trace_id: Optional[str] = None, -): - """ - Helper function to compare actual Langfuse request body with expected JSON file. - - Args: - actual_request_body (dict): The actual request body received from the API call - expected_file_name (str): Name of the JSON file containing expected request body (e.g., "transcription.json") - """ - # Get the current directory and read the expected request body - pwd = os.path.dirname(os.path.realpath(__file__)) - expected_body_path = os.path.join( - pwd, "langfuse_expected_request_body", expected_file_name - ) - - with open(expected_body_path, "r") as f: - expected_request_body = json.load(f) - - # Filter out events that don't match the trace_id - if trace_id: - actual_request_body["batch"] = [ - item - for item in actual_request_body["batch"] - if (item["type"] == "trace-create" and item["body"].get("id") == trace_id) - or ( - item["type"] == "generation-create" - and item["body"].get("traceId") == trace_id - ) - ] - - # When aggregating from multiple flush cycles, deduplicate by keeping - # only one trace-create and one generation-create per trace_id. - seen_types: dict = {} - deduped_batch: list = [] - for item in actual_request_body["batch"]: - item_type = item["type"] - if item_type not in seen_types: - seen_types[item_type] = True - deduped_batch.append(item) - actual_request_body["batch"] = deduped_batch - - # Ensure canonical order: trace-create first, generation-create second - actual_request_body["batch"].sort( - key=lambda x: 0 if x["type"] == "trace-create" else 1 - ) - - print( - "actual_request_body after filtering", json.dumps(actual_request_body, indent=4) - ) - - assert len(actual_request_body["batch"]) >= 2, ( - f"Expected at least 2 batch items (trace-create + generation-create) " - f"after filtering by trace_id={trace_id}, " - f"but got {len(actual_request_body['batch'])}. " - f"Items: {json.dumps(actual_request_body['batch'], indent=2)}" - ) - - # Replace dynamic values in actual request body - for item in actual_request_body["batch"]: - - # Replace IDs with expected IDs - if item["type"] == "trace-create": - item["id"] = expected_request_body["batch"][0]["id"] - item["body"]["id"] = expected_request_body["batch"][0]["body"]["id"] - item["timestamp"] = expected_request_body["batch"][0]["timestamp"] - item["body"]["timestamp"] = expected_request_body["batch"][0]["body"][ - "timestamp" - ] - elif item["type"] == "generation-create": - item["id"] = expected_request_body["batch"][1]["id"] - item["body"]["id"] = expected_request_body["batch"][1]["body"]["id"] - item["timestamp"] = expected_request_body["batch"][1]["timestamp"] - item["body"]["startTime"] = expected_request_body["batch"][1]["body"][ - "startTime" - ] - item["body"]["endTime"] = expected_request_body["batch"][1]["body"][ - "endTime" - ] - item["body"]["completionStartTime"] = expected_request_body["batch"][1][ - "body" - ]["completionStartTime"] - if trace_id is None: - print("popping traceId") - item["body"].pop("traceId") - else: - item["body"]["traceId"] = trace_id - expected_request_body["batch"][1]["body"]["traceId"] = trace_id - - # Replace SDK version with expected version - actual_request_body["batch"][0]["body"].pop("release", None) - actual_request_body["metadata"]["sdk_version"] = expected_request_body["metadata"][ - "sdk_version" - ] - # replace "public_key" with expected public key - actual_request_body["metadata"]["public_key"] = expected_request_body["metadata"][ - "public_key" - ] - actual_request_body["batch"][1]["body"]["metadata"] = expected_request_body[ - "batch" - ][1]["body"]["metadata"] - actual_request_body["metadata"]["sdk_integration"] = expected_request_body[ - "metadata" - ]["sdk_integration"] - actual_request_body["metadata"]["batch_size"] = expected_request_body["metadata"][ - "batch_size" - ] - # Assert the entire request body matches - assert ( - actual_request_body == expected_request_body - ), f"Difference in request bodies: {json.dumps(actual_request_body, indent=2)} != {json.dumps(expected_request_body, indent=2)}" - - -class TestLangfuseLogging: - @pytest_asyncio.fixture - async def mock_setup(self): - """Common setup for Langfuse logging tests""" - from litellm._uuid import uuid - from unittest.mock import AsyncMock, patch - import httpx - - # Create a mock Response object - mock_response = AsyncMock(spec=httpx.Response) - mock_response.status_code = 200 - mock_response.json.return_value = {"status": "success"} - - # Create mock for httpx.Client.post - mock_post = AsyncMock() - mock_post.return_value = mock_response - - litellm.set_verbose = True - litellm.success_callback = ["langfuse"] - - return {"trace_id": f"litellm-test-{str(uuid.uuid4())}", "mock_post": mock_post} - - async def _verify_langfuse_call( - self, - mock_post, - expected_file_name: str, - trace_id: str, - ): - """Helper method to verify Langfuse API calls""" - await asyncio.sleep(3) - - # Verify at least one call was made - assert mock_post.call_count >= 1 - - # Aggregate batch items from ALL calls — the Langfuse SDK may split - # trace-create and generation-create across separate HTTP flushes. - langfuse_url = "https://us.cloud.langfuse.com/api/public/ingestion" - all_batch_items: list = [] - metadata: Optional[dict] = None - for call in mock_post.call_args_list: - url = call[0][0] - if url != langfuse_url: - continue - request_body = call[1].get("content") - if request_body: - body = json.loads(request_body) - all_batch_items.extend(body.get("batch", [])) - if metadata is None: - metadata = body.get("metadata") - - assert len(all_batch_items) > 0, "No Langfuse ingestion calls found" - assert metadata is not None, "No metadata found in Langfuse calls" - - actual_request_body = { - "batch": all_batch_items, - "metadata": metadata, - } - - print("\nMocked Request Details (aggregated from all calls):") - print(f"Request Body: {json.dumps(actual_request_body, indent=4)}") - - assert_langfuse_request_matches_expected( - actual_request_body, - expected_file_name, - trace_id, - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion(self, mock_setup): - """Test Langfuse logging for chat completion""" - setup = mock_setup - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata={"trace_id": setup["trace_id"]}, - ) - await self._verify_langfuse_call( - setup["mock_post"], "completion.json", setup["trace_id"] - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_tags(self, mock_setup): - """Test Langfuse logging for chat completion with tags""" - setup = mock_setup - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata={ - "trace_id": setup["trace_id"], - "tags": ["test_tag", "test_tag_2"], - }, - ) - await self._verify_langfuse_call( - setup["mock_post"], "completion_with_tags.json", setup["trace_id"] - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_tags_stream(self, mock_setup): - """Test Langfuse logging for chat completion with tags""" - setup = mock_setup - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata={ - "trace_id": setup["trace_id"], - "tags": ["test_tag_stream", "test_tag_2_stream"], - }, - ) - await self._verify_langfuse_call( - setup["mock_post"], - "completion_with_tags_stream.json", - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_langfuse_metadata(self, mock_setup): - """Test Langfuse logging for chat completion with metadata for langfuse""" - setup = mock_setup - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata={ - "trace_id": setup["trace_id"], - "tags": ["test_tag", "test_tag_2"], - "generation_name": "test_generation_name", - "parent_observation_id": "test_parent_observation_id", - "version": "test_version", - "trace_user_id": "test_user_id", - "session_id": "test_session_id", - "trace_name": "test_trace_name", - "trace_metadata": {"test_key": "test_value"}, - "trace_version": "test_trace_version", - "trace_release": "test_trace_release", - }, - ) - await self._verify_langfuse_call( - setup["mock_post"], - "completion_with_langfuse_metadata.json", - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_with_non_serializable_metadata(self, mock_setup): - """Test Langfuse logging with metadata that requires preparation (Pydantic models, sets, etc)""" - from pydantic import BaseModel - from typing import Set - import datetime - - class UserPreferences(BaseModel): - favorite_colors: Set[str] - last_login: datetime.datetime - settings: dict - - setup = mock_setup - - test_metadata = { - "user_prefs": UserPreferences( - favorite_colors={"red", "blue"}, - last_login=datetime.datetime.now(), - settings={"theme": "dark", "notifications": True}, - ), - "nested_set": { - "inner_set": {1, 2, 3}, - "inner_pydantic": UserPreferences( - favorite_colors={"green", "yellow"}, - last_login=datetime.datetime.now(), - settings={"theme": "light"}, - ), - }, - "trace_id": setup["trace_id"], - } - - with patch("httpx.Client.post", setup["mock_post"]): - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata=test_metadata, - ) - - await self._verify_langfuse_call( - setup["mock_post"], - "completion_with_complex_metadata.json", - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "test_metadata, response_json_file", - [ - ({"a": 1, "b": 2, "c": 3}, "simple_metadata.json"), - ( - {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, - "nested_metadata.json", - ), - ({"a": [1, 2, 3], "b": {4, 5, 6}}, "simple_metadata2.json"), - ( - {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, - "simple_metadata3.json", - ), - ({"lock": threading.Lock()}, "metadata_with_lock.json"), - ({"func": lambda x: x + 1}, "metadata_with_function.json"), - ( - { - "int": 42, - "str": "hello", - "list": [1, 2, 3], - "set": {4, 5}, - "dict": {"nested": "value"}, - "non_copyable": threading.Lock(), - "function": print, - }, - "complex_metadata.json", - ), - ( - {"list": ["list", "not", "a", "dict"]}, - "complex_metadata_2.json", - ), - ({}, "empty_metadata.json"), - ], - ) - @pytest.mark.flaky(retries=6, delay=1) - async def test_langfuse_logging_with_various_metadata_types( - self, mock_setup, test_metadata, response_json_file - ): - """Test Langfuse logging with various metadata types including non-serializable objects""" - import threading - - setup = mock_setup - - if test_metadata is not None: - test_metadata["trace_id"] = setup["trace_id"] - - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response="Hello! How can I assist you today?", - metadata=test_metadata, - ) - - await self._verify_langfuse_call( - setup["mock_post"], - response_json_file, - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_malformed_llm_response( - self, mock_setup - ): - """Test Langfuse logging for chat completion with malformed LLM response""" - setup = mock_setup - litellm._turn_on_debug() - with patch("httpx.Client.post", setup["mock_post"]): - mock_response = litellm.ModelResponse( - choices=[], - usage=litellm.Usage( - prompt_tokens=10, - completion_tokens=10, - total_tokens=20, - ), - model="gpt-3.5-turbo", - object="chat.completion", - created=1723081200, - ).model_dump() - await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response=mock_response, - metadata={"trace_id": setup["trace_id"]}, - ) - await self._verify_langfuse_call( - setup["mock_post"], "completion_with_no_choices.json", setup["trace_id"] - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_bedrock_llm_response( - self, mock_setup - ): - """Test Langfuse logging for chat completion with malformed LLM response""" - setup = mock_setup - litellm._turn_on_debug() - with patch("httpx.Client.post", setup["mock_post"]): - mock_response = litellm.ModelResponse( - choices=[], - usage=litellm.Usage( - prompt_tokens=10, - completion_tokens=10, - total_tokens=20, - ), - model="anthropic.claude-haiku-4-5-20251001-v1:0", - object="chat.completion", - created=1723081200, - ).model_dump() - await litellm.acompletion( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - messages=[{"role": "user", "content": "Hello!"}], - mock_response=mock_response, - metadata={"trace_id": setup["trace_id"]}, - aws_access_key_id="fake-key", - aws_secret_access_key="fake-key", - aws_region="us-east-1", - ) - await self._verify_langfuse_call( - setup["mock_post"], - "completion_with_bedrock_call.json", - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_completion_with_vertex_llm_response( - self, mock_setup - ): - """Test Langfuse logging for chat completion with malformed LLM response""" - setup = mock_setup - litellm._turn_on_debug() - with patch("httpx.Client.post", setup["mock_post"]): - mock_response = litellm.ModelResponse( - choices=[], - usage=litellm.Usage( - prompt_tokens=10, - completion_tokens=10, - total_tokens=20, - ), - model="vertex/gemini-2.0-flash-001", - object="chat.completion", - created=1723081200, - ).model_dump() - await litellm.acompletion( - model="vertex_ai/gemini-2.0-flash-001", - messages=[{"role": "user", "content": "Hello!"}], - mock_response=mock_response, - metadata={"trace_id": setup["trace_id"]}, - vertex_credentials="my-mock-credentials", - api_key="my-mock-credentials-2", - ) - await self._verify_langfuse_call( - setup["mock_post"], - "completion_with_vertex_call.json", - setup["trace_id"], - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_vllm_embedding(self, mock_setup): - """ - Test that the request sent to the vllm embedding endpoint is correct. - - Verifies the request body matches the expected JSON fixture, - including that the hosted_vllm/ prefix is stripped from the model name - and that no unexpected fields (e.g. encoding_format) are included. - """ - setup = mock_setup - - vllm_response_data = { - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "BAAI/bge-small-en-v1.5", - "usage": {"prompt_tokens": 10, "total_tokens": 10}, - } - mock_vllm_response = httpx.Response( - status_code=200, - json=vllm_response_data, - ) - - mock_async_client = AsyncHTTPHandler() - mock_async_client.post = AsyncMock(return_value=mock_vllm_response) - - with patch("httpx.Client.post", setup["mock_post"]): - await litellm.aembedding( - model="hosted_vllm/BAAI/bge-small-en-v1.5", - input=["Hello from litellm!"], - api_base="http://my-fake-vllm.com/v1", - metadata={"trace_id": setup["trace_id"]}, - client=mock_async_client, - ) - - # Verify the request sent to vllm matches the expected JSON fixture - assert mock_async_client.post.call_count == 1 - actual_vllm_request = mock_async_client.post.call_args.kwargs["json"] - - pwd = os.path.dirname(os.path.realpath(__file__)) - expected_body_path = os.path.join( - pwd, "langfuse_expected_request_body", "embedding_with_vllm.json" - ) - with open(expected_body_path, "r") as f: - expected_vllm_request = json.load(f) - - assert actual_vllm_request == expected_vllm_request, ( - f"vllm request body mismatch:\n" - f"actual: {json.dumps(actual_vllm_request, indent=2)}\n" - f"expected: {json.dumps(expected_vllm_request, indent=2)}" - ) - - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=1) - async def test_langfuse_logging_with_router(self, mock_setup): - """Test Langfuse logging with router""" - litellm._turn_on_debug() - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "mock_response": "Hello! How can I assist you today?", - "api_key": "test_api_key", - }, - } - ] - ) - with patch("httpx.Client.post", mock_setup["mock_post"]): - mock_response = litellm.ModelResponse( - choices=[], - usage=litellm.Usage( - prompt_tokens=10, - completion_tokens=10, - total_tokens=20, - ), - model="gpt-3.5-turbo", - object="chat.completion", - created=1723081200, - ).model_dump() - await router.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello!"}], - mock_response=mock_response, - metadata={"trace_id": mock_setup["trace_id"]}, - ) - await self._verify_langfuse_call( - mock_setup["mock_post"], - "completion_with_router.json", - mock_setup["trace_id"], - )