revert potential breakage where lexicographical sorting is done. Simplify logic for sorting.

Author: erip

test/test_vocab.py

@@ -261,6 +261,17 @@ def test_vocab_specials(self):
         self.assertEqual(v2.get_itos(), expected_itos)
         self.assertEqual(dict(v2.get_stoi()), expected_stoi)
 
+    def test_build_vocab_sorts_descending_frequency_then_lexigraphically(self):
+        it = [["a", "b"], ["a", "b"]]
+        vocab = build_vocab_from_iterator(it)
+        self.assertEqual(vocab["a"], 0)
+        self.assertEqual(vocab["b"], 1)
+
+        it = [["a", "b"], ["b"]]
+        vocab = build_vocab_from_iterator(it)
+        self.assertEqual(vocab["b"], 0)
+        self.assertEqual(vocab["a"], 1)
+
     def test_build_vocab_from_iterator_max_tokens(self):
         it = [["hello", "world"], ["hello"]]
         max_tokens = 1

torchtext/vocab/vocab_factory.py

@@ -93,11 +93,15 @@ def build_vocab_from_iterator(iterator: Iterable, min_freq: int = 1, specials: O
         counter.update(tokens)
 
     specials = specials or []
+
+    # First sort by descending frequency, then lexicographically
+    sorted_by_freq_tuples = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
+
     if max_tokens is None:
-        ordered_dict = OrderedDict(counter.most_common())
+        ordered_dict = OrderedDict(sorted_by_freq_tuples)
     else:
         assert len(specials) < max_tokens, "len(specials) >= max_tokens, so the vocab will be entirely special tokens."
-        ordered_dict = OrderedDict(counter.most_common(max_tokens - len(specials)))
+        ordered_dict = OrderedDict(sorted_by_freq_tuples[:max_tokens - len(specials)])
 
     word_vocab = vocab(ordered_dict, min_freq=min_freq, specials=specials, special_first=special_first)
     return word_vocab