diff --git a/.circleci/config.yml b/.circleci/config.yml index d99c485af94..8672561f654 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1255,7 +1255,15 @@ jobs: ls # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging - python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + # Subdirectories with dedicated jobs (maintain this list as new jobs are added) + IGNORE_DIRS=( + "tests/llm_translation/realtime" + ) + IGNORE_ARGS="" + for dir in "${IGNORE_DIRS[@]}"; do + IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" + done + python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread no_output_timeout: 120m - run: name: Rename the coverage files @@ -1271,6 +1279,54 @@ jobs: paths: - llm_translation_coverage.xml - llm_translation_coverage + realtime_translation_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + pip install "pytest-xdist==3.6.1" + pip install "pytest-timeout==2.2.0" + pip install "websockets" + # Run pytest and generate JUnit XML report + - run: + name: Run realtime tests + command: | + pwd + ls + # Add --timeout to kill hanging tests after 120s (2 min) + # Add --durations=20 to show 20 slowest tests for debugging + python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml realtime_translation_coverage.xml + mv .coverage realtime_translation_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - realtime_translation_coverage.xml + - realtime_translation_coverage mcp_testing: docker: - image: cimg/python:3.11 @@ -3532,7 +3588,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage + coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -4196,6 +4252,12 @@ workflows: only: - main - /litellm_.*/ + - realtime_translation_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - mcp_testing: filters: branches: @@ -4307,6 +4369,7 @@ workflows: - upload-coverage: requires: - llm_translation_testing + - realtime_translation_testing - mcp_testing - google_generate_content_endpoint_testing - guardrails_testing @@ -4384,6 +4447,7 @@ workflows: - e2e_openai_endpoints - test_bad_database_url - llm_translation_testing + - realtime_translation_testing - mcp_testing - google_generate_content_endpoint_testing - llm_responses_api_testing