diff --git a/litellm/proxy/common_utils/periodic_reload_schedule.py b/litellm/proxy/common_utils/periodic_reload_schedule.py index 26ae6bbc4a7..1712376fb14 100644 --- a/litellm/proxy/common_utils/periodic_reload_schedule.py +++ b/litellm/proxy/common_utils/periodic_reload_schedule.py @@ -80,8 +80,17 @@ class _IntervalConfig(BaseModel): interval_hours: int | None = None +def to_db_precision(value: datetime) -> datetime: + """ + Drop sub-millisecond digits, matching the TIMESTAMP(3) columns these values round-trip + through. Comparing a microsecond-precision stamp against its own truncated copy would + read as newer and skip the request it recorded + """ + return value.replace(microsecond=(value.microsecond // 1000) * 1000) + + def utc_now() -> datetime: - return datetime.now(timezone.utc) + return to_db_precision(datetime.now(timezone.utc)) def _parse_interval_hours(param_value: object) -> int | None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a28f39c7a86..4f75763061e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -333,6 +333,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( record_manual_reload, record_reload_run, reload_schedule_status, + to_db_precision, utc_now, write_reload_interval, ) @@ -3862,7 +3863,10 @@ class ProxyConfig: get_model_cost_map_loaded_at, ) - self.model_cost_map_loaded_at: datetime = get_model_cost_map_loaded_at() or utc_now() + cost_map_loaded_at = get_model_cost_map_loaded_at() + self.model_cost_map_loaded_at: datetime = ( + to_db_precision(cost_map_loaded_at) if cost_map_loaded_at is not None else utc_now() + ) def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): diff --git a/tests/test_litellm/proxy/common_utils/test_periodic_reload_schedule.py b/tests/test_litellm/proxy/common_utils/test_periodic_reload_schedule.py index ab28776ee29..e282059eac2 100644 --- a/tests/test_litellm/proxy/common_utils/test_periodic_reload_schedule.py +++ b/tests/test_litellm/proxy/common_utils/test_periodic_reload_schedule.py @@ -13,6 +13,8 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( record_manual_reload, record_reload_run, reload_schedule_status, + to_db_precision, + utc_now, write_reload_interval, ) @@ -134,6 +136,32 @@ def test_pod_reload_is_due(schedule, pod_data_loaded_at, expected): ) +def test_stamps_match_the_precision_they_are_stored_at(): + """Postgres stores TIMESTAMP(3), so a microsecond-precision pod clock would read as + newer than its own persisted copy and skip the request it just recorded""" + assert utc_now().microsecond % 1000 == 0 + assert to_db_precision(datetime(2024, 1, 1, 6, 0, 0, 123456, tzinfo=timezone.utc)) == datetime( + 2024, 1, 1, 6, 0, 0, 123000, tzinfo=timezone.utc + ) + + +def test_request_in_the_same_millisecond_is_not_lost_to_truncation(): + """A pod whose data loaded microseconds before a request must still see it: both sides + are floored to the stored resolution, so the request never reads as older than it is""" + loaded_at = to_db_precision(datetime(2024, 1, 1, 6, 0, 0, 123400, tzinfo=timezone.utc)) + requested_at = datetime(2024, 1, 1, 6, 0, 0, 124000, tzinfo=timezone.utc) + + assert ( + pod_reload_is_due( + schedule=ReloadSchedule(reload_requested_at=requested_at), + pod_data_loaded_at=loaded_at, + current_time=NOW, + description="test", + ) + is True + ) + + def test_manual_request_reaches_every_pod_with_older_data(): """Nothing clears reload_requested_at, so each pod reloads exactly once per request: due while its data predates the request, not due once refreshed or booted after it"""