diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py index b736d2a5d72..2fd8ecf54de 100644 --- a/litellm/llms/soniox/common_utils.py +++ b/litellm/llms/soniox/common_utils.py @@ -2,6 +2,7 @@ Shared utilities for the Soniox provider (https://soniox.com). """ +import unicodedata from typing import Any, Final from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -119,7 +120,35 @@ _CUE_MAX_DURATION_MS: Final[int] = 7000 _CUE_GAP_MS: Final[int] = 700 -_SENTENCE_END_CHARS = (".", "!", "?", "。", "！", "？") +_SENTENCE_END_CHARS = (".", "!", "?", "。", "！", "？", "؟", "۔", "।", "॥", "։", "።") + +_CJK_RANGES = ( + (0x3400, 0x4DBF), + (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), + (0x3040, 0x309F), + (0x30A0, 0x30FF), + (0x31F0, 0x31FF), +) + +_CJK_NO_BREAK_BEFORE = "、。，．！？：；・ー…」』）〉》】〕" + +_CJK_NO_BREAK_AFTER = "「『（〈《【〔" + + +def _is_cjk(ch: str) -> bool: + cp = ord(ch) + return any(lo <= cp <= hi for lo, hi in _CJK_RANGES) + + +def _is_cjk_word_boundary(prev_ch: str, next_ch: str) -> bool: + if not (_is_cjk(prev_ch) or _is_cjk(next_ch)): + return False + return next_ch not in _CJK_NO_BREAK_BEFORE and prev_ch not in _CJK_NO_BREAK_AFTER + + +def _text_width(text: str) -> int: + return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in text) def _format_timestamp_srt(ms: int) -> str: @@ -151,7 +180,10 @@ def _merge_tokens_into_words(tokens: list[dict[str, Any]]) -> list[dict[str, Any Merge Soniox subword tokens (e.g. ``"Hel"``, ``"lo"``) into whole words. A token starts a new word when its text begins with whitespace, when the - previous token's text ends with whitespace, or when the speaker changes. + previous token's text ends with whitespace, when the speaker changes, or + at a CJK character boundary (CJK scripts carry no spaces, so without this + an entire utterance would fuse into a single unbreakable "word"; CJK + punctuation stays attached to the preceding character per kinsoku rules). Each word carries the first/last available timestamps of its tokens. Translation tokens (``translation_status == "translation"``) are excluded: @@ -170,6 +202,7 @@ def _merge_tokens_into_words(tokens: list[dict[str, Any]]) -> list[dict[str, Any and not text[0].isspace() and not words[-1]["text"][-1:].isspace() and token.get("speaker") == words[-1]["speaker"] + and not _is_cjk_word_boundary(words[-1]["text"][-1:], text[0]) ) if is_continuation: last = words[-1] @@ -206,8 +239,8 @@ def _group_tokens_into_cues( - the speaker changes (if diarization is on), - a silence gap of at least _CUE_GAP_MS separates two words, so subtitles never bridge pauses in speech, - - adding the next word would exceed _CUE_MAX_CHARS (~two subtitle - lines), or + - adding the next word would exceed _CUE_MAX_CHARS of display width + (~two subtitle lines; East-Asian wide characters count double), or - the cue would span more than _CUE_MAX_DURATION_MS. A cue also ends after sentence-final punctuation, which keeps cue breaks at natural seams. Cue timestamps come straight from token timestamps; @@ -242,7 +275,7 @@ def _group_tokens_into_cues( w["speaker"] is not None and w["speaker"] != word["speaker"] for w in current ) gap_exceeded = start_ms is not None and cue_end is not None and (start_ms - cue_end) >= _CUE_GAP_MS - chars_exceeded = len(_cue_text(current)) + len(word["text"]) > _CUE_MAX_CHARS + chars_exceeded = _text_width(_cue_text(current)) + _text_width(word["text"]) > _CUE_MAX_CHARS duration_exceeded = ( start_ms is not None and cue_start is not None and (start_ms - cue_start) >= _CUE_MAX_DURATION_MS ) diff --git a/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py b/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py index 45626945bd8..2917d5663ac 100644 --- a/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py +++ b/tests/test_litellm/llms/soniox/audio_transcription/test_soniox_audio_transcription_transformation.py @@ -467,6 +467,91 @@ class TestCueGroupingAlignment: assert "00:00:00,100 --> 00:00:00,500" in result +def _cue_texts(srt: str) -> list: + return [cue.split("\n", 2)[2] for cue in srt.strip().split("\n\n")] + + +class TestMultilingualCueGrouping: + def test_should_split_spaceless_chinese_on_width_budget(self): + from litellm.llms.soniox.common_utils import _text_width, render_soniox_tokens_as_srt + + tokens = [{"text": "你好", "start_ms": i * 100, "end_ms": i * 100 + 90} for i in range(60)] + result = render_soniox_tokens_as_srt(tokens) + texts = _cue_texts(result) + assert len(texts) >= 3 + for text in texts: + assert _text_width(text) <= 84 + assert set(text) <= {"你", "好"} + + def test_should_split_japanese_after_sentence_end_and_keep_punctuation_attached(self): + from litellm.llms.soniox.common_utils import render_soniox_tokens_as_srt + + tokens = [ + {"text": "今日は", "start_ms": 0, "end_ms": 300}, + {"text": "いい", "start_ms": 300, "end_ms": 500}, + {"text": "天気です", "start_ms": 500, "end_ms": 900}, + {"text": "。", "start_ms": 900, "end_ms": 950}, + {"text": "明日も", "start_ms": 1000, "end_ms": 1300}, + {"text": "晴れ", "start_ms": 1300, "end_ms": 1500}, + ] + texts = _cue_texts(render_soniox_tokens_as_srt(tokens)) + assert texts == ["今日はいい天気です。", "明日も晴れ"] + + def test_should_split_arabic_after_arabic_question_mark(self): + from litellm.llms.soniox.common_utils import render_soniox_tokens_as_srt + + tokens = [ + {"text": " كيف", "start_ms": 0, "end_ms": 300}, + {"text": " حالك؟", "start_ms": 300, "end_ms": 700}, + {"text": " أنا", "start_ms": 800, "end_ms": 1000}, + {"text": " بخير", "start_ms": 1000, "end_ms": 1300}, + ] + texts = _cue_texts(render_soniox_tokens_as_srt(tokens)) + assert texts == ["كيف حالك؟", "أنا بخير"] + + def test_should_split_after_devanagari_and_urdu_terminators(self): + from litellm.llms.soniox.common_utils import render_soniox_tokens_as_srt + + tokens = [ + {"text": " नमस्ते।", "start_ms": 0, "end_ms": 400}, + {"text": " آپ", "start_ms": 500, "end_ms": 700}, + {"text": " ٹھیک۔", "start_ms": 700, "end_ms": 1100}, + {"text": " शुभ", "start_ms": 1200, "end_ms": 1400}, + ] + texts = _cue_texts(render_soniox_tokens_as_srt(tokens)) + assert texts == ["नमस्ते।", "آپ ٹھیک۔", "शुभ"] + + def test_should_split_russian_after_sentence_end(self): + from litellm.llms.soniox.common_utils import render_soniox_tokens_as_srt + + tokens = [ + {"text": " Как", "start_ms": 0, "end_ms": 200}, + {"text": " дела?", "start_ms": 200, "end_ms": 600}, + {"text": " Хорошо.", "start_ms": 700, "end_ms": 1200}, + ] + texts = _cue_texts(render_soniox_tokens_as_srt(tokens)) + assert texts == ["Как дела?", "Хорошо."] + + def test_should_not_split_latin_text_within_width_budget(self): + from litellm.llms.soniox.common_utils import render_soniox_tokens_as_srt + + tokens = [{"text": f" word{i}", "start_ms": i * 100, "end_ms": i * 100 + 90} for i in range(12)] + texts = _cue_texts(render_soniox_tokens_as_srt(tokens)) + assert len(texts) == 1 + + def test_should_merge_subword_tokens_inside_cjk_run(self): + from litellm.llms.soniox.common_utils import _merge_tokens_into_words + + tokens = [ + {"text": "編", "start_ms": 0, "end_ms": 100}, + {"text": "集", "start_ms": 100, "end_ms": 200}, + {"text": "、", "start_ms": 200, "end_ms": 250}, + {"text": "保存", "start_ms": 250, "end_ms": 400}, + ] + words = _merge_tokens_into_words(tokens) + assert [w["text"] for w in words] == ["編", "集、", "保存"] + + class TestRenderSonioxTokensAsVtt: def test_should_render_basic_vtt_with_header(self): from litellm.llms.soniox.common_utils import render_soniox_tokens_as_vtt