From d65ba87014a673bca3a4ee1ec4227208eca2ddc8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:19:34 -0700 Subject: [PATCH 01/11] add litellm router testing --- .circleci/config.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 850415b9d7e..1a66a17a54a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,6 +146,33 @@ jobs: python -m pytest -vv tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m + # Store test results + - store_test_results: + path: test-results + litellm_router_testing: + docker: + - image: cimg/python:3.11 + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv -k "router and not openrouter" -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + # Store test results - store_test_results: path: test-results @@ -604,6 +631,12 @@ workflows: only: - main - /litellm_.*/ + - litellm_router_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - ui_endpoint_testing: filters: branches: @@ -645,6 +678,9 @@ workflows: - local_testing - build_and_test - load_testing + - litellm_router_testing + - ui_endpoint_testing + - installing_litellm_on_python - proxy_log_to_otel_tests - proxy_pass_through_endpoint_tests filters: From d944bd98b9b604277c9cacddaa6ff014c20f4b4e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:21:05 -0700 Subject: [PATCH 02/11] fix config.yml --- .circleci/config.yml | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1a66a17a54a..90637db5b01 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -152,27 +152,26 @@ jobs: litellm_router_testing: docker: - image: cimg/python:3.11 - working_directory: ~/project - - steps: - - checkout - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest -vv -k "router and not openrouter" -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m + working_directory: ~/project + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv -k "router and not openrouter" -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m # Store test results - store_test_results: path: test-results From 9c79c1c7b2ba33b998c70aebe1f2ae12e7b0125e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:23:56 -0700 Subject: [PATCH 03/11] fix ci/cd tests --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90637db5b01..a0525041f49 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,7 +105,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py" + python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router" no_output_timeout: 120m # Store test results @@ -149,7 +149,7 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_router_testing: + litellm_router_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 working_directory: ~/project @@ -170,7 +170,7 @@ jobs: command: | pwd ls - python -m pytest -vv -k "router and not openrouter" -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results - store_test_results: From fbe92df87e54893b0ab9c0e7ccf0f7673608e9eb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:24:37 -0700 Subject: [PATCH 04/11] fix router tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0525041f49..18c595384a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,7 +170,7 @@ jobs: command: | pwd ls - python -m pytest -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest litellm/tests/ -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results - store_test_results: From f880e2b958e48bd7215630e4d1ce25ac3be55083 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:26:31 -0700 Subject: [PATCH 05/11] fix respx --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 18c595384a0..07a247c9e25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -162,6 +162,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt pip install "pytest==7.3.1" + pip install "respx==0.21.1" pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" # Run pytest and generate JUnit XML report From e5a776dc07329409d88014f969940e266af12afb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:30:21 -0700 Subject: [PATCH 06/11] make separate assistants testing pipeline --- .circleci/config.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07a247c9e25..53a3e541f34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,7 +105,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router" + python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants" no_output_timeout: 120m # Store test results @@ -176,6 +176,33 @@ jobs: # Store test results - store_test_results: path: test-results + litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword + docker: + - image: cimg/python:3.11 + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.21.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest litellm/tests/ -vv -k "assistants" -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + # Store test results + - store_test_results: + path: test-results load_testing: docker: - image: cimg/python:3.11 @@ -631,6 +658,12 @@ workflows: only: - main - /litellm_.*/ + - litellm_assistants_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - litellm_router_testing: filters: branches: @@ -679,6 +712,7 @@ workflows: - build_and_test - load_testing - litellm_router_testing + - litellm_assistants_api_testing - ui_endpoint_testing - installing_litellm_on_python - proxy_log_to_otel_tests From d038568be421f93dcecc2fc802397d40b05962ab Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:31:17 -0700 Subject: [PATCH 07/11] ci/cd run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 83cb58d3be0..b3b34ac5fe3 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -24,7 +24,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries=3 +# litellm.num_retries= 3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From 9ca7de58d6b0b43c731368952ca9411c848ad93c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:42:00 -0700 Subject: [PATCH 08/11] fix testing --- .circleci/config.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53a3e541f34..c55a8870d3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,7 +105,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants" + python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants and not whisper and not speech" no_output_timeout: 120m # Store test results @@ -203,6 +203,33 @@ jobs: # Store test results - store_test_results: path: test-results + litellm_transcriptions_speech_api_testing: # Runs all tests with the "assistants" keyword + docker: + - image: cimg/python:3.11 + working_directory: ~/project + + steps: + - checkout + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "respx==0.21.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-asyncio==0.21.1" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest litellm/tests/ -vv -k "whisper and speech" -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + # Store test results + - store_test_results: + path: test-results load_testing: docker: - image: cimg/python:3.11 @@ -664,6 +691,12 @@ workflows: only: - main - /litellm_.*/ + - litellm_transcriptions_speech_api_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - litellm_router_testing: filters: branches: @@ -713,6 +746,7 @@ workflows: - load_testing - litellm_router_testing - litellm_assistants_api_testing + - litellm_transcriptions_speech_api_testing - ui_endpoint_testing - installing_litellm_on_python - proxy_log_to_otel_tests From 90d096b6390c970ba8f284862248e20bc8fdd287 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 08:42:34 -0700 Subject: [PATCH 09/11] ci/cd run again --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index b3b34ac5fe3..3adf3bbee9b 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -24,7 +24,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries= 3 +# litellm.num_retries = 3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From bea34c92316588fbf6034f54fb1615ec03845a11 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 09:28:45 -0700 Subject: [PATCH 10/11] fix config.yml --- .circleci/config.yml | 36 +------------------------------- litellm/tests/test_completion.py | 2 +- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c55a8870d3c..53a3e541f34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,7 +105,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants and not whisper and not speech" + python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants" no_output_timeout: 120m # Store test results @@ -203,33 +203,6 @@ jobs: # Store test results - store_test_results: path: test-results - litellm_transcriptions_speech_api_testing: # Runs all tests with the "assistants" keyword - docker: - - image: cimg/python:3.11 - working_directory: ~/project - - steps: - - checkout - - run: - name: Install Dependencies - command: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - pip install "pytest==7.3.1" - pip install "respx==0.21.1" - pip install "pytest-retry==1.6.3" - pip install "pytest-asyncio==0.21.1" - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - pwd - ls - python -m pytest litellm/tests/ -vv -k "whisper and speech" -x -s -v --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: 120m - # Store test results - - store_test_results: - path: test-results load_testing: docker: - image: cimg/python:3.11 @@ -691,12 +664,6 @@ workflows: only: - main - /litellm_.*/ - - litellm_transcriptions_speech_api_testing: - filters: - branches: - only: - - main - - /litellm_.*/ - litellm_router_testing: filters: branches: @@ -746,7 +713,6 @@ workflows: - load_testing - litellm_router_testing - litellm_assistants_api_testing - - litellm_transcriptions_speech_api_testing - ui_endpoint_testing - installing_litellm_on_python - proxy_log_to_otel_tests diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 3adf3bbee9b..eda2b1595a0 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -24,7 +24,7 @@ from litellm import RateLimitError, Timeout, completion, completion_cost, embedd from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.prompt_templates.factory import anthropic_messages_pt -# litellm.num_retries = 3 +# litellm.num_retries =3 litellm.cache = None litellm.success_callback = [] user_message = "Write a short poem about the sky" From 085e1751ad41955b283339d3a4f81d4dd569491e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 12 Sep 2024 09:29:37 -0700 Subject: [PATCH 11/11] mark test as flaky --- litellm/tests/test_streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 7a56d0703ff..2c4190382bb 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -1431,6 +1431,7 @@ async def test_completion_replicate_llama3_streaming(sync_mode): ], ) @pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) async def test_bedrock_httpx_streaming(sync_mode, model, region): try: litellm.set_verbose = True