From 203947df064f02c4c15e64a664ee648bdc6bf711 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 Jan 2025 10:11:04 -0800 Subject: [PATCH] detailed e2e langfuse metadata tests --- .../complex_metadata.json | 101 ++++++++++++++++++ .../complex_metadata_2.json | 93 ++++++++++++++++ .../empty_metadata.json | 87 +++++++++++++++ .../metadata_with_function.json | 87 +++++++++++++++ .../metadata_with_lock.json | 87 +++++++++++++++ .../nested_metadata.json | 93 ++++++++++++++++ .../simple_metadata.json | 93 ++++++++++++++++ .../simple_metadata2.json | 97 +++++++++++++++++ .../simple_metadata3.json | 101 ++++++++++++++++++ .../test_langfuse_e2e_test.py | 61 +++++++++++ .../test_langfuse_unit_tests.py | 46 -------- 11 files changed, 900 insertions(+), 46 deletions(-) create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json new file mode 100644 index 00000000000..01dcd264883 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -0,0 +1,101 @@ +{ + "batch": [ + { + "id": "ddf567e5-a1b5-4e38-8a7c-f48bc847f721", + "type": "trace-create", + "body": { + "id": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", + "timestamp": "2025-01-22T17:59:39.367430Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:39.367707Z" + }, + { + "id": "d3eb2c9e-e123-419d-b27b-c8283a505ae8", + "type": "generation-create", + "body": { + "traceId": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:39.362554-08:00", + "metadata": { + "int": 42, + "str": "hello", + "list": [ + 1, + 2, + 3 + ], + "set": [ + 4, + 5 + ], + "dict": { + "nested": "value" + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-39-362554_chatcmpl-d20ba1d9-cda6-4773-822e-921ebcd426a0", + "endTime": "2025-01-22T09:59:39.365756-08:00", + "completionStartTime": "2025-01-22T09:59:39.365756-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:39.368310Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json new file mode 100644 index 00000000000..1b7b91930e9 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "ea3d694a-ce6b-417e-86e3-23ac17c6f6c6", + "type": "trace-create", + "body": { + "id": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", + "timestamp": "2025-01-22T18:06:50.959206Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T18:06:50.959409Z" + }, + { + "id": "5fe03133-5798-4f87-8eec-ae0264f1eccc", + "type": "generation-create", + "body": { + "traceId": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", + "name": "litellm-acompletion", + "startTime": "2025-01-22T10:06:50.957097-08:00", + "metadata": { + "list": [ + "list", + "not", + "a", + "dict" + ], + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-10-06-50-957097_chatcmpl-62d4ad7c-291b-4fc7-a8a4-3ed0fc3912a5", + "endTime": "2025-01-22T10:06:50.958374-08:00", + "completionStartTime": "2025-01-22T10:06:50.958374-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T18:06:50.959850Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json new file mode 100644 index 00000000000..8c1711ee98e --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "28d0c943-284b-4151-bf0d-8acf0f449865", + "type": "trace-create", + "body": { + "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "timestamp": "2025-01-22T17:59:32.888622Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:32.888940Z" + }, + { + "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", + "type": "generation-create", + "body": { + "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:32.878577-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", + "endTime": "2025-01-22T09:59:32.880691-08:00", + "completionStartTime": "2025-01-22T09:59:32.880691-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:32.889548Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json new file mode 100644 index 00000000000..0b1309425e3 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "88b1898a-cc5d-4e8e-93bc-3e71300c5e8d", + "type": "trace-create", + "body": { + "id": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", + "timestamp": "2025-01-22T17:59:36.162545Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:36.162702Z" + }, + { + "id": "96bb77a6-a350-431b-bfd8-425491259728", + "type": "generation-create", + "body": { + "traceId": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:36.161090-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-36-161090_chatcmpl-1ee988c9-9133-4655-bbe4-b97ffb6e3dc9", + "endTime": "2025-01-22T09:59:36.161959-08:00", + "completionStartTime": "2025-01-22T09:59:36.161959-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:36.162997Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json new file mode 100644 index 00000000000..8c1711ee98e --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "28d0c943-284b-4151-bf0d-8acf0f449865", + "type": "trace-create", + "body": { + "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "timestamp": "2025-01-22T17:59:32.888622Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:32.888940Z" + }, + { + "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", + "type": "generation-create", + "body": { + "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:32.878577-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", + "endTime": "2025-01-22T09:59:32.880691-08:00", + "completionStartTime": "2025-01-22T09:59:32.880691-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:32.889548Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json new file mode 100644 index 00000000000..bb24688aa5c --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "44f179be-e3b9-486f-986f-030fc50614f0", + "type": "trace-create", + "body": { + "id": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", + "timestamp": "2025-01-22T17:55:28.854927Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:55:28.855187Z" + }, + { + "id": "2175ee64-58a3-41ab-96df-405b76695f5f", + "type": "generation-create", + "body": { + "traceId": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:55:28.852503-08:00", + "metadata": { + "a": { + "nested_a": 1 + }, + "b": { + "nested_b": 2 + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-55-28-852503_chatcmpl-131cf0da-a47b-4cd1-850b-50fa077362ac", + "endTime": "2025-01-22T09:55:28.853979-08:00", + "completionStartTime": "2025-01-22T09:55:28.853979-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:55:28.855732Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json new file mode 100644 index 00000000000..d40ec6bafca --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "02c74119-76b7-4f79-91cb-c55f1495c100", + "type": "trace-create", + "body": { + "id": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", + "timestamp": "2025-01-22T17:53:53.754012Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:53:53.754178Z" + }, + { + "id": "097968e0-52e9-46b5-9e8e-e6e08dd00e72", + "type": "generation-create", + "body": { + "traceId": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:53:53.752422-08:00", + "metadata": { + "a": { + "nested_a": 1 + }, + "b": { + "nested_b": 2 + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-53-53-752422_chatcmpl-e99bc1d3-a393-493f-8afe-4507c0acff15", + "endTime": "2025-01-22T09:53:53.753431-08:00", + "completionStartTime": "2025-01-22T09:53:53.753431-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:53:53.754511Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json new file mode 100644 index 00000000000..610bc461a13 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -0,0 +1,97 @@ +{ + "batch": [ + { + "id": "1a55383a-e6fa-41f9-81fe-e7aa58c55f40", + "type": "trace-create", + "body": { + "id": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", + "timestamp": "2025-01-22T17:56:35.477276Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:56:35.477571Z" + }, + { + "id": "13ba66e8-f72b-4f57-a6cc-57c0be2829b1", + "type": "generation-create", + "body": { + "traceId": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:56:35.474752-08:00", + "metadata": { + "a": [ + 1, + 2, + 3 + ], + "b": [ + 4, + 5, + 6 + ], + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-56-35-474752_chatcmpl-9b152610-3d1e-4731-a84e-d0341ea69a0f", + "endTime": "2025-01-22T09:56:35.476236-08:00", + "completionStartTime": "2025-01-22T09:56:35.476236-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:56:35.478171Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json new file mode 100644 index 00000000000..d21c58fdee4 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -0,0 +1,101 @@ +{ + "batch": [ + { + "id": "7fb1f295-a7af-47af-afbd-e2f2d08280aa", + "type": "trace-create", + "body": { + "id": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", + "timestamp": "2025-01-22T17:56:38.786515Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:56:38.786742Z" + }, + { + "id": "412870bc-fc50-4426-a0dc-9e8b016e14bb", + "type": "generation-create", + "body": { + "traceId": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:56:38.784548-08:00", + "metadata": { + "a": [ + 1, + 2 + ], + "b": [ + 3, + 4 + ], + "c": { + "d": [ + 5, + 6 + ] + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-56-38-784548_chatcmpl-438c8727-86b3-44d9-9b46-42330922cf50", + "endTime": "2025-01-22T09:56:38.785762-08:00", + "completionStartTime": "2025-01-22T09:56:38.785762-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:56:38.787196Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index fbafa79b554..60d25b3340e 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -6,6 +6,7 @@ import os import sys from typing import Any, Optional from unittest.mock import MagicMock, patch +import threading logging.basicConfig(level=logging.DEBUG) sys.path.insert(0, os.path.abspath("../..")) @@ -290,3 +291,63 @@ class TestLangfuseLogging: "completion_with_complex_metadata.json", setup["trace_id"], ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "test_metadata, response_json_file", + [ + ({"a": 1, "b": 2, "c": 3}, "simple_metadata.json"), + ( + {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, + "nested_metadata.json", + ), + ({"a": [1, 2, 3], "b": {4, 5, 6}}, "simple_metadata2.json"), + ( + {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, + "simple_metadata3.json", + ), + ({"lock": threading.Lock()}, "metadata_with_lock.json"), + ({"func": lambda x: x + 1}, "metadata_with_function.json"), + ( + { + "int": 42, + "str": "hello", + "list": [1, 2, 3], + "set": {4, 5}, + "dict": {"nested": "value"}, + "non_copyable": threading.Lock(), + "function": print, + }, + "complex_metadata.json", + ), + ( + {"list": ["list", "not", "a", "dict"]}, + "complex_metadata_2.json", + ), + ({}, "empty_metadata.json"), + ], + ) + async def test_langfuse_logging_with_various_metadata_types( + self, mock_setup, test_metadata, response_json_file + ): + """Test Langfuse logging with various metadata types including non-serializable objects""" + import threading + + setup = await mock_setup + + if test_metadata is not None: + test_metadata["trace_id"] = setup["trace_id"] + + with patch("httpx.Client.post", setup["mock_post"]): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hello! How can I assist you today?", + metadata=test_metadata, + ) + + await self._verify_langfuse_call( + setup["mock_post"], + response_json_file, + setup["trace_id"], + ) diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 5a551dbd7db..5096e7b2d79 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -271,52 +271,6 @@ def test_get_langfuse_logger_for_request_with_cached_logger(): mock_cache.get_cache.assert_called_once() -@pytest.mark.parametrize( - "metadata, expected_metadata", - [ - ({"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}), - ( - {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, - {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, - ), - ({"a": [1, 2, 3], "b": {4, 5, 6}}, {"a": [1, 2, 3], "b": {4, 5, 6}}), - ( - {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, - {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, - ), - ({"lock": threading.Lock()}, {}), - ({"func": lambda x: x + 1}, {}), - ( - { - "int": 42, - "str": "hello", - "list": [1, 2, 3], - "set": {4, 5}, - "dict": {"nested": "value"}, - "non_copyable": threading.Lock(), - "function": print, - }, - { - "int": 42, - "str": "hello", - "list": [1, 2, 3], - "set": {4, 5}, - "dict": {"nested": "value"}, - }, - ), - ( - {"list": ["list", "not", "a", "dict"]}, - {"list": ["list", "not", "a", "dict"]}, - ), - ({}, {}), - (None, None), - ], -) -def test_langfuse_logger_prepare_metadata(metadata, expected_metadata): - result = global_langfuse_logger._prepare_metadata(metadata) - assert result == expected_metadata - - def test_get_langfuse_tags(): """ Test that _get_langfuse_tags correctly extracts tags from the standard logging payload