diff --git a/faster_whisper/audio.py b/faster_whisper/audio.py
index fbecc48..4dd8aae 100644
--- a/faster_whisper/audio.py
+++ b/faster_whisper/audio.py
@@ -6,6 +6,7 @@ system dependencies. FFmpeg does not need to be installed on the system.
 However, the API is quite low-level so we need to manipulate audio frames directly.
 """
 
+import gc
 import io
 import itertools
 
@@ -53,6 +54,11 @@ def decode_audio(
             dtype = array.dtype
             raw_buffer.write(array)
 
+    # It appears that some objects related to the resampler are not freed
+    # unless the garbage collector is manually run.
+    del resampler
+    gc.collect()
+
     audio = np.frombuffer(raw_buffer.getbuffer(), dtype=dtype)
 
     # Convert s16 back to f32.