scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

test: remove live OpenAI fine-tuning job-creation test blocked by platform wind-down (#32933)

Browse source 
OpenAI is winding down self-serve fine-tuning and the org can no longer
create fine-tuning jobs (403 training_not_available; the CI key surfaces
it as a 500 server_error), so test_create_fine_tune_jobs_async fails on
every batches_testing run since 2026-07-11 and reruns never clear it.
The request contract stays covered by the mocked create/list/cancel/
retrieve tests in the same file, and the deleted test's unique
standard_logging_object assertions now run inside
test_mock_openai_create_fine_tune_job.
This commit is contained in:
Mateo Wang 2026-07-11 12:57:37 -07:00 • committed by GitHub
parent 2c1d62ce2b
commit 345d353912
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 16 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

128
tests/batches_tests/test_fine_tuning_api.py
View file

@ -13,13 +13,10 @@ import litellm
litellm.num_retries = 0
import asyncio
import logging
from typing import Optional
import openai
from test_openai_batches_and_files import load_vertex_ai_credentials
from litellm import create_fine_tuning_job
from litellm._logging import verbose_logger
from litellm.llms.vertex_ai.fine_tuning.handler import (
FineTuningJobCreate,
VertexFineTuningAPI,
@ -47,115 +44,6 @@ class TestCustomLogger(CustomLogger):
self.standard_logging_object = kwargs["standard_logging_object"]
async def _acreate_fine_tuning_job_with_propagation_retry(
*, max_attempts: int = 12, initial_delay: float = 1.0, **kwargs
):
"""
Wrap litellm.acreate_fine_tuning_job and retry on the eventual-consistency
400 OpenAI returns when a freshly-uploaded training file isn't yet visible
to the fine-tuning endpoint (`'file-... does not exist'`).
Polling the files-retrieve endpoint or `FileObject.status` doesn't help —
OpenAI's `status` field is deprecated, and the retrieve and fine-tuning
endpoints don't share a consistency model. Retrying the operation itself
is the only reliable signal that propagation has finished.
Total budget with defaults: ~70s across 12 attempts (exp backoff capped at
8s).
"""
delay = initial_delay
last_error: Optional[openai.BadRequestError] = None
for _ in range(max_attempts):
try:
return await litellm.acreate_fine_tuning_job(**kwargs)
except openai.BadRequestError as e:
if "does not exist" not in str(e):
raise
last_error = e
await asyncio.sleep(delay)
delay = min(delay * 1.5, 8.0)
assert last_error is not None
raise last_error
@pytest.mark.asyncio
async def test_create_fine_tune_jobs_async():
try:
custom_logger = TestCustomLogger()
litellm.callbacks = ["datadog", custom_logger]
verbose_logger.setLevel(logging.DEBUG)
file_name = "openai_batch_completions.jsonl"
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)
file_obj = await litellm.acreate_file(
file=open(file_path, "rb"),
purpose="fine-tune",
custom_llm_provider="openai",
)
print("Response from creating file=", file_obj)
create_fine_tuning_response = (
await _acreate_fine_tuning_job_with_propagation_retry(
model="gpt-4o-mini-2024-07-18",
training_file=file_obj.id,
)
)
print(
"response from litellm.create_fine_tuning_job=", create_fine_tuning_response
)
assert create_fine_tuning_response.id is not None
assert create_fine_tuning_response.model == "gpt-4o-mini-2024-07-18"
await asyncio.sleep(2)
_logged_standard_logging_object = custom_logger.standard_logging_object
assert _logged_standard_logging_object is not None
print(
"custom_logger.standard_logging_object=",
json.dumps(_logged_standard_logging_object, indent=4),
)
assert _logged_standard_logging_object["model"] == "gpt-4o-mini-2024-07-18"
assert _logged_standard_logging_object["id"] == create_fine_tuning_response.id
# list fine tuning jobs
print("listing ft jobs")
ft_jobs = await litellm.alist_fine_tuning_jobs(limit=2)
print("response from litellm.list_fine_tuning_jobs=", ft_jobs)
assert len(list(ft_jobs)) > 0
# retrieve fine tuning job
response = await litellm.aretrieve_fine_tuning_job(
fine_tuning_job_id=create_fine_tuning_response.id,
)
print("response from litellm.retrieve_fine_tuning_job=", response)
# delete file
await litellm.afile_delete(
file_id=file_obj.id,
)
# cancel ft job
response = await litellm.acancel_fine_tuning_job(
fine_tuning_job_id=create_fine_tuning_response.id,
)
print("response from litellm.cancel_fine_tuning_job=", response)
assert response.status == "cancelled"
assert response.id == create_fine_tuning_response.id
except openai.RateLimitError:
pass
except Exception as e:
if "Job has already completed" in str(e):
return
else:
pytest.fail(f"Error occurred: {e}")
pass
@pytest.mark.asyncio()
async def test_create_vertex_fine_tune_jobs_mocked():
# Define reusable variables for the test
@ -455,6 +343,9 @@ async def test_mock_openai_create_fine_tune_job():
from openai import AsyncOpenAI
from openai.types.fine_tuning.fine_tuning_job import FineTuningJob, Hyperparameters
custom_logger = TestCustomLogger()
previous_callbacks = litellm.callbacks
litellm.callbacks = [custom_logger]
client = AsyncOpenAI(api_key="fake-api-key")
with patch.object(client.fine_tuning.jobs, "create") as mock_create:
@ -500,6 +391,19 @@ async def test_mock_openai_create_fine_tune_job():
== "ft:gpt-4o-mini-2024-07-18:org:custom_suffix:id"
)
try:
for _ in range(20):
if custom_logger.standard_logging_object is not None:
break
await asyncio.sleep(0.25)
logged = custom_logger.standard_logging_object
assert logged is not None
assert logged["model"] == "gpt-4o-mini-2024-07-18"
assert logged["id"] == response.id
assert logged["call_type"] == "acreate_fine_tuning_job"
finally:
litellm.callbacks = previous_callbacks
@pytest.mark.asyncio
async def test_mock_openai_list_fine_tune_jobs():