heimoshuiyu/whisper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix truncated words list when the replacement character is decoded (#1089)

Browse Source
This commit is contained in:
Guillaume Klein
2023-03-14 17:32:41 +01:00
committed by GitHub
parent ba88b8e1b3
commit 5f9ac653b7
2 changed files with 21 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_tokenizer.py
View File

@@ -12,3 +12,13 @@ def test_tokenizer():
assert gpt2_tokenizer.decode(gpt2_tokens) == text
assert multilingual_tokenizer.decode(multilingual_tokens) == text
assert len(gpt2_tokens) > len(multilingual_tokens)
def test_split_on_unicode():
multilingual_tokenizer = get_tokenizer(multilingual=True)
tokens = [8404, 871, 287, 6, 246, 526, 3210, 20378]
words, word_tokens = multilingual_tokenizer.split_tokens_on_unicode(tokens)
assert words == [" elle", " est", " l", "'", "<EFBFBD>", "é", "rit", "oire"]
assert word_tokens == [[8404], [871], [287], [6], [246], [526], [3210], [20378]]