From faddb352f34c58e163e06a4e6d97dd6b6d30965b Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Thu, 24 Dec 2020 21:01:53 +0100 Subject: [PATCH 1/9] initial draft --- torchaudio/datasets/yesno.py | 85 +++++++++++++++--------------------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 7178b8332c..633b4b68fd 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import List, Tuple, Union +from typing import List, Tuple from torch import Tensor from torch.utils.data import Dataset @@ -11,23 +11,14 @@ extract_archive, ) -URL = "http://www.openslr.org/resources/1/waves_yesno.tar.gz" -FOLDER_IN_ARCHIVE = "waves_yesno" -_CHECKSUMS = { - "http://www.openslr.org/resources/1/waves_yesno.tar.gz": - "962ff6e904d2df1126132ecec6978786" -} - - -def load_yesno_item(fileid: str, path: str, ext_audio: str) -> Tuple[Tensor, int, List[int]]: - # Read label - labels = [int(c) for c in fileid.split("_")] - # Read wav - file_audio = os.path.join(path, fileid + ext_audio) - waveform, sample_rate = torchaudio.load(file_audio) - - return waveform, sample_rate, labels +_RELEASE_CONFIGS = { + "release1": { + "folder_in_archive": "waves_yesno", + "url": "http://www.openslr.org/resources/1/waves_yesno.tar.gz", + "checksum": "30301975fd8c5cac4040c261c0852f57cfa8adbbad2ce78e77e4986957445f27", + } +} class YESNO(Dataset): @@ -43,52 +34,44 @@ class YESNO(Dataset): Whether to download the dataset if it is not found at root path. (default: ``False``). """ - _ext_audio = ".wav" - - def __init__(self, - root: Union[str, Path], - url: str = URL, - folder_in_archive: str = FOLDER_IN_ARCHIVE, - download: bool = False) -> None: - - # Get string representation of 'root' in case Path object is passed - root = os.fspath(root) - + def __init__(self, + root: str, + url: str = _RELEASE_CONFIGS["release1"]["url"], + folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], + download: bool = False): + self._ext_audio=".wav" + self._parse_filesystem(root, url, folder_in_archive, download) + + def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, download: bool): + root = Path(root) archive = os.path.basename(url) - archive = os.path.join(root, archive) + archive = root / archive + self._path = os.path.join(root, folder_in_archive) - if download: if not os.path.isdir(self._path): if not os.path.isfile(archive): checksum = _CHECKSUMS.get(url, None) download_url(url, root, hash_value=checksum, hash_type="md5") extract_archive(archive) - + if not os.path.isdir(self._path): raise RuntimeError( "Dataset not found. Please use `download=True` to download it." ) - + self._walker = sorted(str(p.stem) for p in Path(self._path).glob('*' + self._ext_audio)) - - def __getitem__(self, n: int) -> Tuple[Tensor, int, List[int]]: - """Load the n-th sample from the dataset. - - Args: - n (int): The index of the sample to be loaded - - Returns: - tuple: ``(waveform, sample_rate, labels)`` - """ - fileid = self._walker[n] - item = load_yesno_item(fileid, self._path, self._ext_audio) - - # TODO Upon deprecation, uncomment line below and remove following code - # return item - - waveform, sample_rate, labels = item + + def _load_item(self, fileid: str, path: str, ext_audio: str): + labels = [int(c) for c in fileid.split("_")] + file_audio = os.path.join(path, fileid + ext_audio) + waveform, sample_rate = torchaudio.load(file_audio) return waveform, sample_rate, labels - - def __len__(self) -> int: + + def __getitem__(self, n: int): + fileid = self._walker[n] + item = self._load_item(fileid, self._path, self._ext_audio) + return item + + def __len__(self): return len(self._walker) From a410474d31788efcdfffb706ad50c6c4b49fde1a Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Thu, 24 Dec 2020 21:13:40 +0100 Subject: [PATCH 2/9] style checks and checksum fix --- torchaudio/datasets/yesno.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 633b4b68fd..5af79a6a66 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -34,44 +34,44 @@ class YESNO(Dataset): Whether to download the dataset if it is not found at root path. (default: ``False``). """ - def __init__(self, + def __init__(self, root: str, url: str = _RELEASE_CONFIGS["release1"]["url"], folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], download: bool = False): - self._ext_audio=".wav" + self._ext_audio = ".wav" self._parse_filesystem(root, url, folder_in_archive, download) - + def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, download: bool): root = Path(root) archive = os.path.basename(url) archive = root / archive - + self._path = os.path.join(root, folder_in_archive) if download: if not os.path.isdir(self._path): if not os.path.isfile(archive): - checksum = _CHECKSUMS.get(url, None) + checksum = _RELEASE_CONFIGS["release1"]["checksum"] download_url(url, root, hash_value=checksum, hash_type="md5") extract_archive(archive) - + if not os.path.isdir(self._path): raise RuntimeError( "Dataset not found. Please use `download=True` to download it." ) - + self._walker = sorted(str(p.stem) for p in Path(self._path).glob('*' + self._ext_audio)) - + def _load_item(self, fileid: str, path: str, ext_audio: str): labels = [int(c) for c in fileid.split("_")] file_audio = os.path.join(path, fileid + ext_audio) waveform, sample_rate = torchaudio.load(file_audio) return waveform, sample_rate, labels - + def __getitem__(self, n: int): fileid = self._walker[n] item = self._load_item(fileid, self._path, self._ext_audio) return item - + def __len__(self): return len(self._walker) From db7ac070ea9380b34d3a3c9181256083eae2e697 Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Thu, 24 Dec 2020 21:15:47 +0100 Subject: [PATCH 3/9] indent --- torchaudio/datasets/yesno.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 5af79a6a66..9e01bf613a 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -34,11 +34,14 @@ class YESNO(Dataset): Whether to download the dataset if it is not found at root path. (default: ``False``). """ - def __init__(self, - root: str, - url: str = _RELEASE_CONFIGS["release1"]["url"], - folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], - download: bool = False): + def __init__( + self, + root: str, + url: str = _RELEASE_CONFIGS["release1"]["url"], + folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], + download: bool = False + ) -> None: + self._ext_audio = ".wav" self._parse_filesystem(root, url, folder_in_archive, download) From dc26ae1d00a33d615a365eaddab7f5cadec842cd Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Fri, 25 Dec 2020 20:02:58 +0100 Subject: [PATCH 4/9] add doc string and remove ext --- torchaudio/datasets/yesno.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 9e01bf613a..a6f0ae4a21 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, Union from torch import Tensor from torch.utils.data import Dataset @@ -36,13 +36,12 @@ class YESNO(Dataset): def __init__( self, - root: str, + root: Union[str, Path], url: str = _RELEASE_CONFIGS["release1"]["url"], folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], download: bool = False ) -> None: - self._ext_audio = ".wav" self._parse_filesystem(root, url, folder_in_archive, download) def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, download: bool): @@ -63,17 +62,25 @@ def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, downloa "Dataset not found. Please use `download=True` to download it." ) - self._walker = sorted(str(p.stem) for p in Path(self._path).glob('*' + self._ext_audio)) + self._walker = sorted(str(p.stem) for p in Path(self._path).glob("*.wav")) - def _load_item(self, fileid: str, path: str, ext_audio: str): + def _load_item(self, fileid: str, path: str): labels = [int(c) for c in fileid.split("_")] - file_audio = os.path.join(path, fileid + ext_audio) + file_audio = os.path.join(path, fileid + "*.wav") waveform, sample_rate = torchaudio.load(file_audio) return waveform, sample_rate, labels def __getitem__(self, n: int): + """Load the n-th sample from the dataset. + + Args: + n (int): The index of the sample to be loaded + + Returns: + tuple: ``(waveform, sample_rate, labels)`` + """ fileid = self._walker[n] - item = self._load_item(fileid, self._path, self._ext_audio) + item = self._load_item(fileid, self._path) return item def __len__(self): From 4721d9337456365761608aac4ebcdf3367054e56 Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Fri, 25 Dec 2020 20:04:56 +0100 Subject: [PATCH 5/9] pathlib modifications --- torchaudio/datasets/yesno.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index a6f0ae4a21..08246536c4 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -49,7 +49,7 @@ def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, downloa archive = os.path.basename(url) archive = root / archive - self._path = os.path.join(root, folder_in_archive) + self._path = root / folder_in_archive if download: if not os.path.isdir(self._path): if not os.path.isfile(archive): From f357d3303d71675626cddfe34cfceae8973b43f7 Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Fri, 25 Dec 2020 21:13:13 +0100 Subject: [PATCH 6/9] fix typo --- torchaudio/datasets/yesno.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 08246536c4..cbab560534 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -66,7 +66,7 @@ def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, downloa def _load_item(self, fileid: str, path: str): labels = [int(c) for c in fileid.split("_")] - file_audio = os.path.join(path, fileid + "*.wav") + file_audio = os.path.join(path, fileid + ".wav") waveform, sample_rate = torchaudio.load(file_audio) return waveform, sample_rate, labels From d9920bd92a2227d61fda6d5d48c502b1a159c7fd Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Fri, 25 Dec 2020 21:33:01 +0100 Subject: [PATCH 7/9] fix indent --- torchaudio/datasets/yesno.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index cbab560534..8c797e331e 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -39,8 +39,7 @@ def __init__( root: Union[str, Path], url: str = _RELEASE_CONFIGS["release1"]["url"], folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], - download: bool = False - ) -> None: + download: bool = False) -> None: self._parse_filesystem(root, url, folder_in_archive, download) From e0e0b60eb5b3632e79094d38bacf5e88afb4605e Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Mon, 28 Dec 2020 11:21:26 +0100 Subject: [PATCH 8/9] minor changes --- torchaudio/datasets/yesno.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index 8c797e331e..a0fd8cbc25 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -39,11 +39,12 @@ def __init__( root: Union[str, Path], url: str = _RELEASE_CONFIGS["release1"]["url"], folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], - download: bool = False) -> None: + download: bool = False + ) -> None: self._parse_filesystem(root, url, folder_in_archive, download) - def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, download: bool): + def _parse_filesystem(self, root: str, url: str, folder_in_archive: str, download: bool) -> None: root = Path(root) archive = os.path.basename(url) archive = root / archive @@ -69,7 +70,7 @@ def _load_item(self, fileid: str, path: str): waveform, sample_rate = torchaudio.load(file_audio) return waveform, sample_rate, labels - def __getitem__(self, n: int): + def __getitem__(self, n: int) -> Tuple[Tensor, int, List[int]]: """Load the n-th sample from the dataset. Args: @@ -82,5 +83,5 @@ def __getitem__(self, n: int): item = self._load_item(fileid, self._path) return item - def __len__(self): + def __len__(self) -> int: return len(self._walker) From 3667a6956a20e86d98a097c00f8f38e1c8f24a2e Mon Sep 17 00:00:00 2001 From: krishnakalyan3 Date: Thu, 31 Dec 2020 22:13:47 +0530 Subject: [PATCH 9/9] indent correct --- torchaudio/datasets/yesno.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchaudio/datasets/yesno.py b/torchaudio/datasets/yesno.py index a0fd8cbc25..6d0d2c0a5f 100644 --- a/torchaudio/datasets/yesno.py +++ b/torchaudio/datasets/yesno.py @@ -40,7 +40,7 @@ def __init__( url: str = _RELEASE_CONFIGS["release1"]["url"], folder_in_archive: str = _RELEASE_CONFIGS["release1"]["folder_in_archive"], download: bool = False - ) -> None: + ) -> None: self._parse_filesystem(root, url, folder_in_archive, download)