Automatically download converted models from the Hugging Face Hub (#70)

* Automatically download converted models from the Hugging Face Hub

* Remove unused import

* Remove non needed requirements in dev mode

* Remove extra index URL when pip install in CI

* Allow downloading to a specific directory

* Update docstring

* Add argument to disable the progess bars

* Fix typo in docstring
This commit is contained in:
Guillaume Klein
2023-03-24 10:55:55 +01:00
committed by GitHub
parent 523ae2180f
commit de7682a2f0
10 changed files with 105 additions and 53 deletions

View File

@@ -1,6 +1,5 @@
import os
import ctranslate2
import pytest
@@ -12,20 +11,3 @@ def data_dir():
@pytest.fixture
def jfk_path(data_dir):
return os.path.join(data_dir, "jfk.flac")
@pytest.fixture(scope="session")
def tiny_model_dir(tmp_path_factory):
model_path = str(tmp_path_factory.mktemp("data") / "model")
convert_model("tiny", model_path)
return model_path
def convert_model(size, output_dir):
name = "openai/whisper-%s" % size
ctranslate2.converters.TransformersConverter(
name,
copy_files=["tokenizer.json"],
load_as_float16=True,
).convert(output_dir, quantization="float16")