Remove unnecessary fsspec usage

This commit is contained in:
Agrin Hilmkil 2021-04-28 11:14:00 +02:00
parent bf2b9958be
commit 351d0ed6ae
2 changed files with 4 additions and 7 deletions

View File

@ -25,7 +25,6 @@ import subprocess
import sys import sys
import zipfile import zipfile
import fsspec
import pandas import pandas
import soundfile as sf import soundfile as sf
from absl import logging from absl import logging
@ -72,7 +71,7 @@ def download_and_extract(directory, subset, urls):
subset: subset name of the corpus. subset: subset name of the corpus.
urls: the list of urls to download the data file. urls: the list of urls to download the data file.
""" """
fsspec.get_mapper(directory).fs.makedirs(directory, exist_ok=True) os.makedirs(directory, exist_ok=True)
try: try:
for url in urls: for url in urls:
@ -105,7 +104,7 @@ def download_and_extract(directory, subset, urls):
extract_path_ori = os.path.join(directory, zfile.infolist()[0].filename) extract_path_ori = os.path.join(directory, zfile.infolist()[0].filename)
subprocess.call("mv %s %s" % (extract_path_ori, extract_path), shell=True) subprocess.call("mv %s %s" % (extract_path_ori, extract_path), shell=True)
finally: finally:
# fsspec.get_mapper(directory).fs.rm_file(zip_filepath) # os.remove(zip_filepath)
pass pass
@ -158,8 +157,7 @@ def convert_audio_and_make_label(input_dir, subset, output_dir, output_file):
files = [] files = []
# Convert all AAC file into WAV format. At the same time, generate the csv # Convert all AAC file into WAV format. At the same time, generate the csv
fs = fsspec.get_mapper(source_dir).fs for root, _, filenames in os.walk(source_dir):
for root, _, filenames in fs.walk(source_dir):
for filename in filenames: for filename in filenames:
name, ext = os.path.splitext(filename) name, ext = os.path.splitext(filename)
if ext.lower() == ".wav": if ext.lower() == ".wav":
@ -171,7 +169,7 @@ def convert_audio_and_make_label(input_dir, subset, output_dir, output_file):
# Convert AAC to WAV. # Convert AAC to WAV.
aac_file = os.path.join(root, filename) aac_file = os.path.join(root, filename)
wav_file = aac_file + ".wav" wav_file = aac_file + ".wav"
if not fs.exists(wav_file): if not os.path.exists(wav_file):
if not decode_aac_with_ffmpeg(aac_file, wav_file): if not decode_aac_with_ffmpeg(aac_file, wav_file):
raise RuntimeError("Audio decoding failed.") raise RuntimeError("Audio decoding failed.")
else: else:

View File

@ -1,6 +1,5 @@
cython cython
flask flask
fsspec>=0.8.0
gdown gdown
inflect inflect
jieba jieba