Revert "Prevent out of range error in method split_tokens_on_unicode"

This reverts commit 36160c1e7e.
2023-04-04 10:25:41 +02:00
parent 36160c1e7e
commit 9fa1989073
1 changed files with 0 additions and 3 deletions
--- a/faster_whisper/tokenizer.py
+++ b/faster_whisper/tokenizer.py
@@ -135,9 +135,6 @@ class Tokenizer:
                current_tokens = []
                unicode_offset += len(decoded)

-                if unicode_offset >= len(decoded_full):
-                    break
-
        return words, word_tokens

    def split_tokens_on_spaces(