heimoshuiyu/faster-whisper
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Prevent out of range error in method split_tokens_on_unicode (#111)

Browse Source
This commit is contained in:
Guillaume Klein
2023-04-04 10:51:14 +02:00
committed by GitHub
parent 9fa1989073
commit a5d03e55fa
1 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
faster_whisper/tokenizer.py
View File

@@ -125,10 +125,15 @@ class Tokenizer:
current_tokens.append(token) current_tokens.append(token)
decoded = self.decode_with_timestamps(current_tokens) decoded = self.decode_with_timestamps(current_tokens)
if ( try:
replacement_char not in decoded replacement_char_index = decoded.index(replacement_char)
or decoded_full[unicode_offset + decoded.index(replacement_char)] replacement_char_index += unicode_offset
== replacement_char except ValueError:
replacement_char_index = None
if replacement_char_index is None or (
replacement_char_index < len(decoded_full)
and decoded_full[replacement_char_index] == replacement_char
): ):
words.append(decoded) words.append(decoded)
word_tokens.append(current_tokens) word_tokens.append(current_tokens)