Skip to content

Commit 10b9240

Browse files
prabhat00155facebook-github-bot
authored andcommitted
Added dataset download support in fbcode (#3823)
Summary: Pull Request resolved: #3823 Uploaded FashionMNIST dataset to [manifold](https://www.internalfb.com/intern/network/manifold/?bucket=torchvision&path=tree%2Fdatasets) bucket `torchvision`. Any new dataset that needs to be added could be uploaded under `tree/datasets/<dataset_name>`. Reviewed By: datumbox Differential Revision: D28358470 fbshipit-source-id: 6f2282d3f1ce4b1416e962de8fb132896d4b2d76
1 parent 7bee5ed commit 10b9240

File tree

2 files changed

+32
-17
lines changed

2 files changed

+32
-17
lines changed

torchvision/datasets/_utils.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
def _download_file_from_remote_location(fpath: str) -> None:
2+
pass
3+
4+
5+
def _is_remote_location_available() -> bool:
6+
return False

torchvision/datasets/utils.py

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
import torch
1818
from torch.utils.model_zoo import tqdm
1919

20+
from ._utils import (
21+
_download_file_from_remote_location,
22+
_is_remote_location_available,
23+
)
24+
2025

2126
USER_AGENT = "pytorch/vision"
2227

@@ -117,26 +122,30 @@ def download_url(
117122
print('Using downloaded and verified file: ' + fpath)
118123
return
119124

120-
# expand redirect chain if needed
121-
url = _get_redirect_url(url, max_hops=max_redirect_hops)
125+
if _is_remote_location_available():
126+
_download_file_from_remote_location(fpath)
127+
else:
128+
# expand redirect chain if needed
129+
url = _get_redirect_url(url, max_hops=max_redirect_hops)
122130

123-
# check if file is located on Google Drive
124-
file_id = _get_google_drive_file_id(url)
125-
if file_id is not None:
126-
return download_file_from_google_drive(file_id, root, filename, md5)
131+
# check if file is located on Google Drive
132+
file_id = _get_google_drive_file_id(url)
133+
if file_id is not None:
134+
return download_file_from_google_drive(file_id, root, filename, md5)
127135

128-
# download the file
129-
try:
130-
print('Downloading ' + url + ' to ' + fpath)
131-
_urlretrieve(url, fpath)
132-
except (urllib.error.URLError, IOError) as e: # type: ignore[attr-defined]
133-
if url[:5] == 'https':
134-
url = url.replace('https:', 'http:')
135-
print('Failed download. Trying https -> http instead.'
136-
' Downloading ' + url + ' to ' + fpath)
136+
# download the file
137+
try:
138+
print('Downloading ' + url + ' to ' + fpath)
137139
_urlretrieve(url, fpath)
138-
else:
139-
raise e
140+
except (urllib.error.URLError, IOError) as e: # type: ignore[attr-defined]
141+
if url[:5] == 'https':
142+
url = url.replace('https:', 'http:')
143+
print('Failed download. Trying https -> http instead.'
144+
' Downloading ' + url + ' to ' + fpath)
145+
_urlretrieve(url, fpath)
146+
else:
147+
raise e
148+
140149
# check integrity of downloaded file
141150
if not check_integrity(fpath, md5):
142151
raise RuntimeError("File not found or corrupted.")

0 commit comments

Comments
 (0)