update amazon dataset

Author: parmeet

test/data/test_builtin_datasets.py

@@ -63,8 +63,7 @@ def test_raw_text_classification(self, info):
             return
         else:
             data_iter = torchtext.datasets.DATASETS[dataset_name](split=split)
-        self.assertEqual(len(data_iter), info['NUM_LINES'])
-        self.assertEqual(hashlib.md5(json.dumps(next(data_iter), sort_keys=True).encode('utf-8')).hexdigest(), info['first_line'])
+        self.assertEqual(hashlib.md5(json.dumps(next(iter(data_iter)), sort_keys=True).encode('utf-8')).hexdigest(), info['first_line'])
         if dataset_name == "AG_NEWS":
             self.assertEqual(torchtext.datasets.URLS[dataset_name][split], info['URL'])
             self.assertEqual(torchtext.datasets.MD5[dataset_name][split], info['MD5'])

torchtext/datasets/amazonreviewpolarity.py

@@ -50,4 +50,4 @@ def AmazonReviewPolarity(root, split):
     filter_extracted_files = extracted_files.filter(lambda x: split in x[0])
 
     # stack CSV reader and do some mapping
-    return filter_extracted_files.parse_csv().map(fn=lambda t: (int(t[0]), t[1]))
+    return filter_extracted_files.parse_csv().map(fn=lambda t: (int(t[0]), ' '.join(t[1:])))