diff --git a/litellm/tests/test_helicone_integration.py b/litellm/tests/test_helicone_integration.py index 4b92c1bbdb4..8e1f0a94e93 100644 --- a/litellm/tests/test_helicone_integration.py +++ b/litellm/tests/test_helicone_integration.py @@ -3,6 +3,7 @@ import copy import logging import os import sys +import time from typing import Any from unittest.mock import MagicMock, patch @@ -15,56 +16,10 @@ from litellm import completion litellm.num_retries = 3 litellm.success_callback = ["helicone"] os.environ["HELICONE_DEBUG"] = "True" +os.environ['LITELLM_LOG'] = 'DEBUG' import pytest -def search_logs(log_file_path, num_good_logs=1): - import re - - print("\n searching logs") - bad_logs = [] - good_logs = [] - all_logs = [] - try: - with open(log_file_path, "r") as log_file: - lines = log_file.readlines() - print(f"searching logslines: {lines}") - for line in lines: - all_logs.append(line.strip()) - if "/v1/request/query" in line: - print("Found log with /v1/request/query:") - print(line.strip()) - print("\n\n") - match = re.search( - r'"POST /v1/request/query HTTP/1.1" (\d+) (\d+)', - line, - ) - if match: - status_code = int(match.group(1)) - print("STATUS CODE", status_code) - if status_code != 200: - print("got a BAD log") - bad_logs.append(line.strip()) - else: - good_logs.append(line.strip()) - print("\nBad Logs") - print(bad_logs) - if len(bad_logs) > 0: - raise Exception(f"bad logs, Bad logs = {bad_logs}") - assert ( - len(good_logs) == num_good_logs - ), f"Did not get expected number of good logs, expected {num_good_logs}, got {len(good_logs)}. All logs \n {all_logs}" - print("\nGood Logs") - print(good_logs) - if len(good_logs) <= 0: - raise Exception( - f"There were no Good Logs from Helicone. No logs with /v1/request/query status 200. \nAll logs:{all_logs}" - ) - - except Exception as e: - raise e - - def pre_helicone_setup(): """ Set up the logging for the 'pre_helicone_setup' function. @@ -83,8 +38,6 @@ def pre_helicone_setup(): def test_helicone_logging_async(): try: pre_helicone_setup() - litellm.set_verbose = True - litellm.success_callback = [] start_time_empty_callback = asyncio.run(make_async_calls()) print("done with no callback test") @@ -137,36 +90,35 @@ def create_async_task(**completion_kwargs): return asyncio.create_task(litellm.acompletion(**completion_args)) @pytest.mark.asyncio +@pytest.mark.skipif( + condition=not os.environ.get("OPENAI_API_KEY", False), + reason="Authentication missing for openai", +) async def test_helicone_logging_metadata(): import uuid - - litellm.set_verbose = True litellm.success_callback = ["helicone"] - run_id = str(uuid.uuid4()) - request_id = f"litellm-test-session-{run_id}" + request_id = str(uuid.uuid4()) trace_common_metadata = { "Helicone-Property-Request-Id": request_id } - for request_num in range(1, 3): - metadata = copy.deepcopy(trace_common_metadata) - metadata["Helicone-Property-Conversation"] = "support_issue" - response = await create_async_task( - model="gpt-3.5-turbo", - mock_response=f"{request_id}", - messages=[ - { - "role": "user", - "content": f"{request_id}", - } - ], - max_tokens=100, - temperature=0.2, - metadata=copy.deepcopy(metadata), - ) - print(response) + + metadata = copy.deepcopy(trace_common_metadata) + metadata["Helicone-Property-Conversation"] = "support_issue" + metadata["Helicone-Auth"] = os.getenv("HELICONE_API_KEY") + response = await create_async_task( + model="gpt-3.5-turbo", + mock_response="Hey! how's it going?", + messages=[ + { + "role": "user", + "content": f"{request_id}", + } + ], + max_tokens=100, + temperature=0.2, + metadata=copy.deepcopy(metadata), + ) + print(response) - await asyncio.sleep(2) - - # Check log file for entries - search_logs("helicone.log") + time.sleep(3)