Skip to content

Commit e76e52f

Browse files
NicolasHugpmeierfmassa
authored and
facebook-github-bot
committed
[fbsync] Download and Kinetics 400/600/700 Datasets (#3680)
Summary: * Initial commit * pmeiers comments * pmeiers changes * pmeiers comments * replace pandas with system library to avoid crashes * Lint * Lint * fixing unittest * Minor comments removal * pmeier comments * remove asserts * address pmeier formatting changes * address pmeier changes * pmeier changes * rename n_classes to num_classes * formatting changes * doc change to add ".mp4" to backported class * formatting to correct line length * adding **kwargs to Kinetics400 class * remove urlib request and download the file directly * annotations and files can be already downloaded * test fix * add download tests for Kinetics * users now dont need to provide full path within the root for new Kinetics dataset * linter * Update test/test_datasets_download.py * Update torchvision/datasets/kinetics.py * revert whitespace (3680#discussion_r626382842) * addressing annotation_path parameter which is unnecessary * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * kwargs update * expose num_download_workers as public * swap os.isfile with check_integrity * nit on private things * special case if there are no default arguments * revert changes to kinetics400 test case for BC * add split_folder changes and support for legacy format * pmeiers suggestions * pmeiers suggestions - root comment * pmeiers comments - annotation attribute remmoved * pmeiers suggestion * pmeiers suggestion * pmeiers suggestion * pmeiers suggestion * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * Update torchvision/datasets/kinetics.py * minor debugging * nit picks * only include public kwargs into defaults * add _use_legacy_structure in favour of **kwargs * add type hints for Kinetics400 * flake8 * flake8 * flake8 * rename to make thigs clearer * permuting the output Reviewed By: fmassa Differential Revision: D29097736 fbshipit-source-id: 1de2119e82eadbbba682f0897f03aba3929e3604 Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Philip Meier <[email protected]> Co-authored-by: Francisco Massa <[email protected]>
1 parent 3482225 commit e76e52f

File tree

5 files changed

+305
-30
lines changed

5 files changed

+305
-30
lines changed

‎test/datasets_utils.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,11 @@ def _populate_private_class_attributes(cls):
416416
continue
417417

418418
defaults.append(
419-
{kwarg: default for kwarg, default in zip(argspec.args[-len(argspec.defaults):], argspec.defaults)}
419+
{
420+
kwarg: default
421+
for kwarg, default in zip(argspec.args[-len(argspec.defaults):], argspec.defaults)
422+
if not kwarg.startswith("_")
423+
}
420424
)
421425

422426
if not argspec.varkw:
@@ -637,7 +641,7 @@ def __init__(self, *args, **kwargs):
637641

638642
def _set_default_frames_per_clip(self, inject_fake_data):
639643
argspec = inspect.getfullargspec(self.DATASET_CLASS.__init__)
640-
args_without_default = argspec.args[1:-len(argspec.defaults)]
644+
args_without_default = argspec.args[1:(-len(argspec.defaults) if argspec.defaults else None)]
641645
frames_per_clip_last = args_without_default[-1] == "frames_per_clip"
642646

643647
@functools.wraps(inject_fake_data)

‎test/test_datasets.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,27 @@ def test_not_found_or_corrupted(self):
944944
super().test_not_found_or_corrupted()
945945

946946

947+
class KineticsTestCase(datasets_utils.VideoDatasetTestCase):
948+
DATASET_CLASS = datasets.Kinetics
949+
ADDITIONAL_CONFIGS = datasets_utils.combinations_grid(
950+
split=("train", "val"), num_classes=("400", "600", "700")
951+
)
952+
953+
def inject_fake_data(self, tmpdir, config):
954+
classes = ("Abseiling", "Zumba")
955+
num_videos_per_class = 2
956+
tmpdir = pathlib.Path(tmpdir) / config['split']
957+
digits = string.ascii_letters + string.digits + "-_"
958+
for cls in classes:
959+
datasets_utils.create_video_folder(
960+
tmpdir,
961+
cls,
962+
lambda _: f"{datasets_utils.create_random_string(11, digits)}.mp4",
963+
num_videos_per_class,
964+
)
965+
return num_videos_per_class * len(classes)
966+
967+
947968
class Kinetics400TestCase(datasets_utils.VideoDatasetTestCase):
948969
DATASET_CLASS = datasets.Kinetics400
949970

‎test/test_datasets_download.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,25 @@ def widerface():
392392
)
393393

394394

395+
def kinetics():
396+
return itertools.chain(
397+
*[
398+
collect_download_configs(
399+
lambda: datasets.Kinetics(
400+
path.join(ROOT, f"Kinetics{num_classes}"),
401+
frames_per_clip=1,
402+
num_classes=num_classes,
403+
split=split,
404+
download=True,
405+
),
406+
name=f"Kinetics, {num_classes}, {split}",
407+
file="kinetics",
408+
)
409+
for num_classes, split in itertools.product(("400", "600", "700"), ("train", "val"))
410+
]
411+
)
412+
413+
395414
def kitti():
396415
return itertools.chain(
397416
*[
@@ -440,6 +459,7 @@ def make_parametrize_kwargs(download_configs):
440459
usps(),
441460
celeba(),
442461
widerface(),
462+
kinetics(),
443463
kitti(),
444464
)
445465
)

‎torchvision/datasets/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from .sbd import SBDataset
2121
from .vision import VisionDataset
2222
from .usps import USPS
23-
from .kinetics import Kinetics400
23+
from .kinetics import Kinetics400, Kinetics
2424
from .hmdb51 import HMDB51
2525
from .ucf101 import UCF101
2626
from .places365 import Places365
@@ -34,6 +34,6 @@
3434
'Omniglot', 'SBU', 'Flickr8k', 'Flickr30k',
3535
'VOCSegmentation', 'VOCDetection', 'Cityscapes', 'ImageNet',
3636
'Caltech101', 'Caltech256', 'CelebA', 'WIDERFace', 'SBDataset',
37-
'VisionDataset', 'USPS', 'Kinetics400', 'HMDB51', 'UCF101',
37+
'VisionDataset', 'USPS', 'Kinetics400', "Kinetics", 'HMDB51', 'UCF101',
3838
'Places365', 'Kitti',
3939
)

0 commit comments

Comments
 (0)