From baca675956bab1c58c393a223d55c31ffc56e310 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 00:06:14 -0700 Subject: [PATCH 01/22] Working draft without _source --- Lib/collections/__init__.py | 135 +++++++++++++++++------------------ Lib/test/test_collections.py | 55 +------------- 2 files changed, 65 insertions(+), 125 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 70cb683088be09..f3138462a1b47b 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -301,58 +301,6 @@ def __eq__(self, other): ### namedtuple ################################################################################ -_class_template = """\ -from builtins import property as _property, tuple as _tuple -from operator import itemgetter as _itemgetter -from collections import OrderedDict - -class {typename}(tuple): - '{typename}({arg_list})' - - __slots__ = () - - _fields = {field_names!r} - - def __new__(_cls, {arg_list}): - 'Create new instance of {typename}({arg_list})' - return _tuple.__new__(_cls, ({arg_list})) - - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new {typename} object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != {num_fields:d}: - raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result)) - return result - - def _replace(_self, **kwds): - 'Return a new {typename} object replacing specified fields with new values' - result = _self._make(map(kwds.pop, {field_names!r}, _self)) - if kwds: - raise ValueError('Got unexpected field names: %r' % list(kwds)) - return result - - def __repr__(self): - 'Return a nicely formatted representation string' - return self.__class__.__name__ + '({repr_fmt})' % self - - def _asdict(self): - 'Return a new OrderedDict which maps field names to their values.' - return OrderedDict(zip(self._fields, self)) - - def __getnewargs__(self): - 'Return self as a plain tuple. Used by copy and pickle.' - return tuple(self) - -{field_defs} -""" - -_repr_template = '{name}=%r' - -_field_template = '''\ - {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}') -''' - def namedtuple(typename, field_names, *, verbose=False, rename=False, module=None): """Returns a new subclass of tuple with named fields. @@ -410,26 +358,71 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non raise ValueError('Encountered duplicate field name: %r' % name) seen.add(name) - # Fill-in the class template - class_definition = _class_template.format( - typename = typename, - field_names = tuple(field_names), - num_fields = len(field_names), - arg_list = repr(tuple(field_names)).replace("'", "")[1:-1], - repr_fmt = ', '.join(_repr_template.format(name=name) - for name in field_names), - field_defs = '\n'.join(_field_template.format(index=index, name=name) - for index, name in enumerate(field_names)) - ) + # Closure variables used in the methods and docstrings + field_names = tuple(field_names) + num_fields = len(field_names) + arg_list = repr(field_names).replace("'", "")[1:-1] + repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' + tuple_new = tuple.__new__ + _len = len + + # Create all the named tuple methods to be added to the class namespace + + def create_new(): + s = f'def __new__(_cls, {arg_list}): return _tuple.__new__(_cls, ({arg_list}))' + namespace = dict(_tuple=tuple, __name__=f'namedtuple_{typename}') + exec(s, namespace) + __new__ = namespace['__new__'] + __new__.__doc__ = f'Create new instance of {typename}({arg_list})' + return __new__ + + def create_make(): + def _make(cls, iterable): + result = tuple_new(cls, iterable) + if _len(result) != num_fields: + raise TypeError(f'Expected {num_fields:d} arguments, got {len(result)}') + return result + _make.__doc__ = f'Make a new {typename} object from a sequence or iterable' + return classmethod(_make) + + def create_replace(): + def _replace(_self, **kwds): + result = _self._make(map(kwds.pop, field_names, _self)) + if kwds: + raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + return result + _replace.__doc__ = f'Return a new {typename} object replacing specified fields with new values' + return _replace + + def __repr__(self): + 'Return a nicely formatted representation string' + return self.__class__.__name__ + repr_fmt % self + + def _asdict(self): + 'Return a new OrderedDict which maps field names to their values.' + return OrderedDict(zip(self._fields, self)) - # Execute the template string in a temporary namespace and support - # tracing utilities by setting a value for frame.f_globals['__name__'] - namespace = dict(__name__='namedtuple_%s' % typename) - exec(class_definition, namespace) - result = namespace[typename] - result._source = class_definition - if verbose: - print(result._source) + def __getnewargs__(self): + 'Return self as a plain tuple. Used by copy and pickle.' + return tuple(self) + + # Build-up the class namespace dictionary + # and use type() to build the result class + class_namespace = dict( + __slots__ = (), + __doc__ = f'{typename}({arg_list})', + _fields = field_names, + __new__ = create_new(), + _make = create_make(), + _replace = create_replace(), + __repr__ = __repr__, + _asdict = _asdict, + __getnewargs__ = __getnewargs__, + ) + for index, name in enumerate(field_names): + class_namespace[name] = property(fget=_itemgetter(index), + doc=f'Alias for field number {index:d}') + result = type(typename, (tuple,), class_namespace) # For pickling to work, the __module__ variable needs to be set to the frame # where the named tuple is created. Bypass this step in environments where diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 3bf15786189b30..b480aa5473224e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -194,7 +194,7 @@ def test_factory(self): self.assertEqual(Point.__module__, __name__) self.assertEqual(Point.__getitem__, tuple.__getitem__) self.assertEqual(Point._fields, ('x', 'y')) - self.assertIn('class Point(tuple)', Point._source) + #self.assertIn('class Point(tuple)', Point._source) self.assertRaises(ValueError, namedtuple, 'abc%', 'efg ghi') # type has non-alpha char self.assertRaises(ValueError, namedtuple, 'class', 'efg ghi') # type has keyword @@ -366,35 +366,6 @@ def test_name_conflicts(self): newt = t._replace(itemgetter=10, property=20, self=30, cls=40, tuple=50) self.assertEqual(newt, (10,20,30,40,50)) - # Broader test of all interesting names in a template - with support.captured_stdout() as template: - T = namedtuple('T', 'x', verbose=True) - words = set(re.findall('[A-Za-z]+', template.getvalue())) - words -= set(keyword.kwlist) - T = namedtuple('T', words) - # test __new__ - values = tuple(range(len(words))) - t = T(*values) - self.assertEqual(t, values) - t = T(**dict(zip(T._fields, values))) - self.assertEqual(t, values) - # test _make - t = T._make(values) - self.assertEqual(t, values) - # exercise __repr__ - repr(t) - # test _asdict - self.assertEqual(t._asdict(), dict(zip(T._fields, values))) - # test _replace - t = T._make(values) - newvalues = tuple(v*10 for v in values) - newt = t._replace(**dict(zip(T._fields, newvalues))) - self.assertEqual(newt, newvalues) - # test _fields - self.assertEqual(T._fields, tuple(words)) - # test __getnewargs__ - self.assertEqual(t.__getnewargs__(), values) - def test_repr(self): with support.captured_stdout() as template: A = namedtuple('A', 'x', verbose=True) @@ -404,30 +375,6 @@ class B(A): pass self.assertEqual(repr(B(1)), 'B(x=1)') - def test_source(self): - # verify that _source can be run through exec() - tmp = namedtuple('NTColor', 'red green blue') - globals().pop('NTColor', None) # remove artifacts from other tests - exec(tmp._source, globals()) - self.assertIn('NTColor', globals()) - c = NTColor(10, 20, 30) - self.assertEqual((c.red, c.green, c.blue), (10, 20, 30)) - self.assertEqual(NTColor._fields, ('red', 'green', 'blue')) - globals().pop('NTColor', None) # clean-up after this test - - def test_keyword_only_arguments(self): - # See issue 25628 - with support.captured_stdout() as template: - NT = namedtuple('NT', ['x', 'y'], verbose=True) - self.assertIn('class NT', NT._source) - with self.assertRaises(TypeError): - NT = namedtuple('NT', ['x', 'y'], True) - - NT = namedtuple('NT', ['abc', 'def'], rename=True) - self.assertEqual(NT._fields, ('abc', '_1')) - with self.assertRaises(TypeError): - NT = namedtuple('NT', ['abc', 'def'], False, True) - def test_namedtuple_subclass_issue_24931(self): class Point(namedtuple('_Point', ['x', 'y'])): pass From 3a2aa415618ef45af68f21afbd1ce56f95693c9c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 01:06:50 -0700 Subject: [PATCH 02/22] Re-use itemgetter() instances --- Lib/collections/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index f3138462a1b47b..325aee20534490 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -301,6 +301,8 @@ def __eq__(self, other): ### namedtuple ################################################################################ +_nt_itemgetters = {} + def namedtuple(typename, field_names, *, verbose=False, rename=False, module=None): """Returns a new subclass of tuple with named fields. @@ -406,6 +408,13 @@ def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + def reuse_itemgetter(index): + try: + return _nt_itemgetters[index] + except KeyError: + getter = _nt_itemgetters[index] = _itemgetter(index) + return getter + # Build-up the class namespace dictionary # and use type() to build the result class class_namespace = dict( @@ -420,8 +429,8 @@ def __getnewargs__(self): __getnewargs__ = __getnewargs__, ) for index, name in enumerate(field_names): - class_namespace[name] = property(fget=_itemgetter(index), - doc=f'Alias for field number {index:d}') + class_namespace[name] = property(fget = reuse_itemgetter(index), + doc = f'Alias for field number {index}') result = type(typename, (tuple,), class_namespace) # For pickling to work, the __module__ variable needs to be set to the frame From e24ef4beb1cbb90639af13bc09a367bf54c7a48c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 01:21:06 -0700 Subject: [PATCH 03/22] Speed-up calls to __new__() with a pre-bound tuple.__new__() --- Lib/collections/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 325aee20534490..f4a7e8cdd373c4 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -371,8 +371,8 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non # Create all the named tuple methods to be added to the class namespace def create_new(): - s = f'def __new__(_cls, {arg_list}): return _tuple.__new__(_cls, ({arg_list}))' - namespace = dict(_tuple=tuple, __name__=f'namedtuple_{typename}') + s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' + namespace = dict(_tuple_new=tuple_new, __name__=f'namedtuple_{typename}') exec(s, namespace) __new__ = namespace['__new__'] __new__.__doc__ = f'Create new instance of {typename}({arg_list})' From 49aa967264c56110863bf87acb5c2303c4c61c01 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 02:00:18 -0700 Subject: [PATCH 04/22] Add note regarding string interning --- Lib/collections/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index f4a7e8cdd373c4..12f32c674cc2e2 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -373,6 +373,7 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non def create_new(): s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' namespace = dict(_tuple_new=tuple_new, __name__=f'namedtuple_{typename}') + # Note: exec() has the side-effect of interning the typename and field names exec(s, namespace) __new__ = namespace['__new__'] __new__.__doc__ = f'Create new instance of {typename}({arg_list})' From c34b44488feb94901b5c224989d72bbddb1d9786 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 09:51:06 -0700 Subject: [PATCH 05/22] Remove unnecessary create function wrappers --- Lib/collections/__init__.py | 57 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 12f32c674cc2e2..b35daaa346b3d6 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -370,32 +370,29 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non # Create all the named tuple methods to be added to the class namespace - def create_new(): - s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' - namespace = dict(_tuple_new=tuple_new, __name__=f'namedtuple_{typename}') - # Note: exec() has the side-effect of interning the typename and field names - exec(s, namespace) - __new__ = namespace['__new__'] - __new__.__doc__ = f'Create new instance of {typename}({arg_list})' - return __new__ - - def create_make(): - def _make(cls, iterable): - result = tuple_new(cls, iterable) - if _len(result) != num_fields: - raise TypeError(f'Expected {num_fields:d} arguments, got {len(result)}') - return result - _make.__doc__ = f'Make a new {typename} object from a sequence or iterable' - return classmethod(_make) - - def create_replace(): - def _replace(_self, **kwds): - result = _self._make(map(kwds.pop, field_names, _self)) - if kwds: - raise ValueError(f'Got unexpected field names: {list(kwds)!r}') - return result - _replace.__doc__ = f'Return a new {typename} object replacing specified fields with new values' - return _replace + s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' + namespace = dict(_tuple_new=tuple_new, __name__=f'namedtuple_{typename}') + # Note: exec() has the side-effect of interning the typename and field names + exec(s, namespace) + __new__ = namespace['__new__'] + __new__.__doc__ = f'Create new instance of {typename}({arg_list})' + + @classmethod + def _make(cls, iterable): + result = tuple_new(cls, iterable) + if _len(result) != num_fields: + raise TypeError(f'Expected {num_fields:d} arguments, got {len(result)}') + return result + + _make.__doc__ = f'Make a new {typename} object from a sequence or iterable' + + def _replace(_self, **kwds): + result = _self._make(map(kwds.pop, field_names, _self)) + if kwds: + raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + return result + + _replace.__doc__ = f'Return a new {typename} object replacing specified fields with new values' def __repr__(self): 'Return a nicely formatted representation string' @@ -419,12 +416,12 @@ def reuse_itemgetter(index): # Build-up the class namespace dictionary # and use type() to build the result class class_namespace = dict( - __slots__ = (), __doc__ = f'{typename}({arg_list})', + __slots__ = (), _fields = field_names, - __new__ = create_new(), - _make = create_make(), - _replace = create_replace(), + __new__ = __new__, + _make = _make, + _replace = _replace, __repr__ = __repr__, _asdict = _asdict, __getnewargs__ = __getnewargs__, From 0c7f16393c33350809396cc6c3491c1e767f0805 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 10:25:31 -0700 Subject: [PATCH 06/22] Minor sync-ups with PR-2736. Mostly formatting and f-strings --- Lib/collections/__init__.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index b35daaa346b3d6..ecd724ff3b4450 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -340,38 +340,39 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non or _iskeyword(name) or name.startswith('_') or name in seen): - field_names[index] = '_%d' % index + field_names[index] = f'_{index}' seen.add(name) for name in [typename] + field_names: if type(name) is not str: raise TypeError('Type names and field names must be strings') if not name.isidentifier(): raise ValueError('Type names and field names must be valid ' - 'identifiers: %r' % name) + f'identifiers: {name!r}') if _iskeyword(name): raise ValueError('Type names and field names cannot be a ' - 'keyword: %r' % name) + f'keyword: {name!r}') seen = set() for name in field_names: if name.startswith('_') and not rename: raise ValueError('Field names cannot start with an underscore: ' - '%r' % name) + f'{name!r}') if name in seen: - raise ValueError('Encountered duplicate field name: %r' % name) + raise ValueError('Encountered duplicate field name: {name!r}') seen.add(name) - # Closure variables used in the methods and docstrings + # Variables used in the methods and docstrings field_names = tuple(field_names) num_fields = len(field_names) arg_list = repr(field_names).replace("'", "")[1:-1] repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' + module_name = f'namedtuple_{typename}' tuple_new = tuple.__new__ _len = len # Create all the named tuple methods to be added to the class namespace s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' - namespace = dict(_tuple_new=tuple_new, __name__=f'namedtuple_{typename}') + namespace = dict(_tuple_new=tuple_new, __name__=module_name) # Note: exec() has the side-effect of interning the typename and field names exec(s, namespace) __new__ = namespace['__new__'] @@ -381,7 +382,7 @@ def namedtuple(typename, field_names, *, verbose=False, rename=False, module=Non def _make(cls, iterable): result = tuple_new(cls, iterable) if _len(result) != num_fields: - raise TypeError(f'Expected {num_fields:d} arguments, got {len(result)}') + raise TypeError(f'Expected {num_fields} arguments, got {len(result)}') return result _make.__doc__ = f'Make a new {typename} object from a sequence or iterable' @@ -392,7 +393,8 @@ def _replace(_self, **kwds): raise ValueError(f'Got unexpected field names: {list(kwds)!r}') return result - _replace.__doc__ = f'Return a new {typename} object replacing specified fields with new values' + _replace.__doc__ = (f'Return a new {typename} object replacing specified ' + 'fields with new values') def __repr__(self): 'Return a nicely formatted representation string' @@ -406,6 +408,8 @@ def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + # Helper functions used in the class creation + def reuse_itemgetter(index): try: return _nt_itemgetters[index] @@ -427,8 +431,9 @@ def reuse_itemgetter(index): __getnewargs__ = __getnewargs__, ) for index, name in enumerate(field_names): - class_namespace[name] = property(fget = reuse_itemgetter(index), - doc = f'Alias for field number {index}') + doc = f'Alias for field number {index}' + class_namespace[name] = property(reuse_itemgetter(index), doc=doc) + result = type(typename, (tuple,), class_namespace) # For pickling to work, the __module__ variable needs to be set to the frame From 5de0b7fd80bf9db74ea136b4d16d1cfb5a2aff28 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 10:36:21 -0700 Subject: [PATCH 07/22] Bring-in qualname/__module fix-ups from PR-2736 --- Lib/collections/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index ecd724ff3b4450..bd97e7104477fa 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -408,6 +408,12 @@ def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + # Modify function metadata to help with introspection and debugging + + for method in (__new__, _make.__func__, _replace, __repr__, _asdict, __getnewargs__): + method.__module__ = module_name + method.__qualname__ = f'{typename}.{method.__name__}' + # Helper functions used in the class creation def reuse_itemgetter(index): From ca643f4acb542c64cb6aae58ce06b5d4e11f6e7c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 11:52:02 -0700 Subject: [PATCH 08/22] Formally remove the verbose flag and _source attribute --- Doc/library/collections.rst | 18 ++++-------------- Lib/collections/__init__.py | 5 +++-- Lib/test/test_collections.py | 4 +--- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index d6d2056dfc496c..7d6e2422a0d6d6 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -763,7 +763,7 @@ Named tuples assign meaning to each position in a tuple and allow for more reada self-documenting code. They can be used wherever regular tuples are used, and they add the ability to access fields by name instead of position index. -.. function:: namedtuple(typename, field_names, *, verbose=False, rename=False, module=None) +.. function:: namedtuple(typename, field_names, *, rename=False, module=None) Returns a new tuple subclass named *typename*. The new subclass is used to create tuple-like objects that have fields accessible by attribute lookup as @@ -786,10 +786,6 @@ they add the ability to access fields by name instead of position index. converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword ``def`` and the duplicate fieldname ``abc``. - If *verbose* is true, the class definition is printed after it is - built. This option is outdated; instead, it is simpler to print the - :attr:`_source` attribute. - If *module* is defined, the ``__module__`` attribute of the named tuple is set to that value. @@ -806,6 +802,9 @@ they add the ability to access fields by name instead of position index. .. versionchanged:: 3.6 Added the *module* parameter. + .. versionchanged:: 3.7 + Remove the *verbose* parameter and the :attr:`_source` attribute. + .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -878,15 +877,6 @@ field names, the method and attribute names start with an underscore. >>> for partnum, record in inventory.items(): ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) -.. attribute:: somenamedtuple._source - - A string with the pure Python source code used to create the named - tuple class. The source makes the named tuple self-documenting. - It can be printed, executed using :func:`exec`, or saved to a file - and imported. - - .. versionadded:: 3.3 - .. attribute:: somenamedtuple._fields Tuple of strings listing the field names. Useful for introspection diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index bd97e7104477fa..6afbc1bc684e44 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -303,7 +303,7 @@ def __eq__(self, other): _nt_itemgetters = {} -def namedtuple(typename, field_names, *, verbose=False, rename=False, module=None): +def namedtuple(typename, field_names, *, rename=False, module=None): """Returns a new subclass of tuple with named fields. >>> Point = namedtuple('Point', ['x', 'y']) @@ -385,7 +385,8 @@ def _make(cls, iterable): raise TypeError(f'Expected {num_fields} arguments, got {len(result)}') return result - _make.__doc__ = f'Make a new {typename} object from a sequence or iterable' + _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence ' + 'or iterable') def _replace(_self, **kwds): result = _self._make(map(kwds.pop, field_names, _self)) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index b480aa5473224e..e05c1a1f75f8c3 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -194,7 +194,6 @@ def test_factory(self): self.assertEqual(Point.__module__, __name__) self.assertEqual(Point.__getitem__, tuple.__getitem__) self.assertEqual(Point._fields, ('x', 'y')) - #self.assertIn('class Point(tuple)', Point._source) self.assertRaises(ValueError, namedtuple, 'abc%', 'efg ghi') # type has non-alpha char self.assertRaises(ValueError, namedtuple, 'class', 'efg ghi') # type has keyword @@ -367,8 +366,7 @@ def test_name_conflicts(self): self.assertEqual(newt, (10,20,30,40,50)) def test_repr(self): - with support.captured_stdout() as template: - A = namedtuple('A', 'x', verbose=True) + A = namedtuple('A', 'x') self.assertEqual(repr(A(1)), 'A(x=1)') # repr should show the name of the subclass class B(A): From e18b92ed04e43b914ddc0f3acf69e5dbef848ef8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 13:24:09 -0700 Subject: [PATCH 09/22] Restore a test of potentially problematic field names --- Doc/library/collections.rst | 2 +- Lib/test/test_collections.py | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 7d6e2422a0d6d6..cda829694a3277 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -804,7 +804,7 @@ they add the ability to access fields by name instead of position index. .. versionchanged:: 3.7 Remove the *verbose* parameter and the :attr:`_source` attribute. - + .. doctest:: :options: +NORMALIZE_WHITESPACE diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index e05c1a1f75f8c3..30b836c4115442 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -365,6 +365,61 @@ def test_name_conflicts(self): newt = t._replace(itemgetter=10, property=20, self=30, cls=40, tuple=50) self.assertEqual(newt, (10,20,30,40,50)) + # Broader test of all interesting names taken from the code, old + # template, and an example + words = {'Alias', 'At', 'AttributeError', 'Build', 'Bypass', 'Create', + 'Encountered', 'Expected', 'Field', 'For', 'Got', 'Helper', + 'IronPython', 'Jython', 'KeyError', 'Make', 'Modify', 'Note', + 'OrderedDict', 'Point', 'Return', 'Returns', 'Type', 'TypeError', + 'Used', 'Validate', 'ValueError', 'Variables', 'a', 'accessible', 'add', + 'added', 'all', 'also', 'an', 'arg_list', 'args', 'arguments', + 'automatically', 'be', 'build', 'builtins', 'but', 'by', 'cannot', + 'class_namespace', 'classmethod', 'cls', 'collections', 'convert', + 'copy', 'created', 'creation', 'd', 'debugging', 'defined', 'dict', + 'dictionary', 'doc', 'docstring', 'docstrings', 'duplicate', 'effect', + 'either', 'enumerate', 'environments', 'error', 'example', 'exec', 'f', + 'f_globals', 'field', 'field_names', 'fields', 'formatted', 'frame', + 'function', 'functions', 'generate', 'get', 'getter', 'got', 'greater', + 'has', 'help', 'identifiers', 'index', 'indexable', 'instance', + 'instantiate', 'interning', 'introspection', 'isidentifier', + 'isinstance', 'itemgetter', 'iterable', 'join', 'keyword', 'keywords', + 'kwds', 'len', 'like', 'list', 'map', 'maps', 'message', 'metadata', + 'method', 'methods', 'module', 'module_name', 'must', 'name', 'named', + 'namedtuple', 'namedtuple_', 'names', 'namespace', 'needs', 'new', + 'nicely', 'num_fields', 'number', 'object', 'of', 'operator', 'option', + 'p', 'particular', 'pickle', 'pickling', 'plain', 'pop', 'positional', + 'property', 'r', 'regular', 'rename', 'replace', 'replacing', 'repr', + 'repr_fmt', 'representation', 'result', 'reuse_itemgetter', 's', 'seen', + 'self', 'sequence', 'set', 'side', 'specified', 'split', 'start', + 'startswith', 'step', 'str', 'string', 'strings', 'subclass', 'sys', + 'targets', 'than', 'the', 'their', 'this', 'to', 'tuple', 'tuple_new', + 'type', 'typename', 'underscore', 'unexpected', 'unpack', 'up', 'use', + 'used', 'user', 'valid', 'values', 'variable', 'verbose', 'where', + 'which', 'work', 'x', 'y', 'z', 'zip'} + T = namedtuple('T', words) + # test __new__ + values = tuple(range(len(words))) + t = T(*values) + self.assertEqual(t, values) + t = T(**dict(zip(T._fields, values))) + self.assertEqual(t, values) + # test _make + t = T._make(values) + self.assertEqual(t, values) + # exercise __repr__ + repr(t) + # test _asdict + self.assertEqual(t._asdict(), dict(zip(T._fields, values))) + # test _replace + t = T._make(values) + newvalues = tuple(v*10 for v in values) + newt = t._replace(**dict(zip(T._fields, newvalues))) + self.assertEqual(newt, newvalues) + # test _fields + self.assertEqual(T._fields, tuple(words)) + # test __getnewargs__ + self.assertEqual(t.__getnewargs__(), values) + def test_repr(self): A = namedtuple('A', 'x') self.assertEqual(repr(A(1)), 'A(x=1)') From 0851fc76dbdeef765ca070208f505e29b7b8d1e8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 13:31:44 -0700 Subject: [PATCH 10/22] Restore kwonly_args test but without the verbose option --- Lib/test/test_collections.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 30b836c4115442..75defa12739b3d 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -428,6 +428,16 @@ class B(A): pass self.assertEqual(repr(B(1)), 'B(x=1)') + def test_keyword_only_arguments(self): + # See issue 25628 + with self.assertRaises(TypeError): + NT = namedtuple('NT', ['x', 'y'], True) + + NT = namedtuple('NT', ['abc', 'def'], rename=True) + self.assertEqual(NT._fields, ('abc', '_1')) + with self.assertRaises(TypeError): + NT = namedtuple('NT', ['abc', 'def'], False, True) + def test_namedtuple_subclass_issue_24931(self): class Point(namedtuple('_Point', ['x', 'y'])): pass From 3ac1151fda5bbfb8da18c7c44b56477fbe71293a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 13:40:12 -0700 Subject: [PATCH 11/22] Adopt Inada's idea to reuse the docstrings for the itemgetters --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 6afbc1bc684e44..9df6a416dc816d 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -438,7 +438,7 @@ def reuse_itemgetter(index): __getnewargs__ = __getnewargs__, ) for index, name in enumerate(field_names): - doc = f'Alias for field number {index}' + doc = _sys.intern(f'Alias for field number {index}') class_namespace[name] = property(reuse_itemgetter(index), doc=doc) result = type(typename, (tuple,), class_namespace) From b31f0639f3f83ca4a7a98715a2693742a97bf8f6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 13:44:47 -0700 Subject: [PATCH 12/22] Neaten-up a bit --- Lib/collections/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 9df6a416dc816d..b172a8a2ad3f1f 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -411,11 +411,12 @@ def __getnewargs__(self): # Modify function metadata to help with introspection and debugging - for method in (__new__, _make.__func__, _replace, __repr__, _asdict, __getnewargs__): + for method in (__new__, _make.__func__, _replace, + __repr__, _asdict, __getnewargs__): method.__module__ = module_name method.__qualname__ = f'{typename}.{method.__name__}' - # Helper functions used in the class creation + # Helper function used in the class creation def reuse_itemgetter(index): try: From deb30a2fe688832de98c2ebfd962d060d54d5ef6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 14:31:58 -0700 Subject: [PATCH 13/22] Add news blurb --- .../Library/2017-09-08-14-31-15.bpo-28638.lfbVyH.rst | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-09-08-14-31-15.bpo-28638.lfbVyH.rst diff --git a/Misc/NEWS.d/next/Library/2017-09-08-14-31-15.bpo-28638.lfbVyH.rst b/Misc/NEWS.d/next/Library/2017-09-08-14-31-15.bpo-28638.lfbVyH.rst new file mode 100644 index 00000000000000..53b809f51c0f9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-08-14-31-15.bpo-28638.lfbVyH.rst @@ -0,0 +1,9 @@ +Changed the implementation strategy for collections.namedtuple() to +substantially reduce the use of exec() in favor of precomputed methods. As a +result, the *verbose* parameter and *_source* attribute are no longer +supported. The benefits include 1) having a smaller memory footprint for +applications using multiple named tuples, 2) faster creation of the named +tuple class (approx 4x to 6x depending on how it is measured), and 3) minor +speed-ups for instance creation using __new__, _make, and _replace. (The +primary patch contributor is Jelle Zijlstra with further improvements by +INADA Naoki, Serhiy Storchaka, and Raymond Hettinger.) From 86cef9e8ef80217dcb93e2f11821683a91930faa Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 21:43:18 -0700 Subject: [PATCH 14/22] Serhiy pointed-out the need for interning --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index b172a8a2ad3f1f..51f8c582d2effd 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -361,7 +361,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): seen.add(name) # Variables used in the methods and docstrings - field_names = tuple(field_names) + field_names = tuple(map(_sys.intern, field_names)) num_fields = len(field_names) arg_list = repr(field_names).replace("'", "")[1:-1] repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' From 83b5e93876ea47b81a476c79767306cb474b9152 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 8 Sep 2017 22:53:21 -0700 Subject: [PATCH 15/22] Jelle noticed as missing f on an f-string --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 51f8c582d2effd..252551a1087322 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -357,7 +357,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): raise ValueError('Field names cannot start with an underscore: ' f'{name!r}') if name in seen: - raise ValueError('Encountered duplicate field name: {name!r}') + raise ValueError(f'Encountered duplicate field name: {name!r}') seen.add(name) # Variables used in the methods and docstrings From 0168317c6ebf19be74990479003eb8d81faf7694 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 10:50:30 -0700 Subject: [PATCH 16/22] Add whatsnew entry for feature removal --- Doc/whatsnew/3.7.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 5ee41dcc20b170..6ff1bfcb68f575 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -435,6 +435,12 @@ API and Feature Removals Python 3.1, and has now been removed. Use the :func:`~os.path.splitdrive` function instead. +* :func:`collections.namedtuple` no longer supports the *verbose* parameter + or ``_source`` attribute which showed the generated source code for the + named tuple class. This was part of an optimization designed to speed-up + class creation. (Contributed by Jelle Zijlstra with further improvements + by INADA Naoki, Serhiy Storchaka, and Raymond Hettinger in :issue:`28638`.) + * Functions :func:`bool`, :func:`float`, :func:`list` and :func:`tuple` no longer take keyword arguments. The first argument of :func:`int` can now be passed only as positional argument. From dadafc06fd0ff8fc9f78511d73ca6b1485b17791 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 12:09:31 -0700 Subject: [PATCH 17/22] Accede to request for dict literals instead keyword arguments --- Lib/collections/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 252551a1087322..d31b4e973ec1a9 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -372,7 +372,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): # Create all the named tuple methods to be added to the class namespace s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' - namespace = dict(_tuple_new=tuple_new, __name__=module_name) + namespace = {'_tuple_new': tuple_new, '__name__': module_name} # Note: exec() has the side-effect of interning the typename and field names exec(s, namespace) __new__ = namespace['__new__'] @@ -427,17 +427,17 @@ def reuse_itemgetter(index): # Build-up the class namespace dictionary # and use type() to build the result class - class_namespace = dict( - __doc__ = f'{typename}({arg_list})', - __slots__ = (), - _fields = field_names, - __new__ = __new__, - _make = _make, - _replace = _replace, - __repr__ = __repr__, - _asdict = _asdict, - __getnewargs__ = __getnewargs__, - ) + class_namespace = { + '__doc__': f'{typename}({arg_list})', + '__slots__': (), + '_fields': field_names, + '__new__': __new__, + '_make': _make, + '_replace': _replace, + '__repr__': __repr__, + '_asdict': _asdict, + '__getnewargs__': __getnewargs__, + } for index, name in enumerate(field_names): doc = _sys.intern(f'Alias for field number {index}') class_namespace[name] = property(reuse_itemgetter(index), doc=doc) From 483d08c4e9b949703a3bc2eae73095b4cdc739b1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 12:12:19 -0700 Subject: [PATCH 18/22] Leave the method.__module__ attribute pointing the actual location of the code --- Lib/collections/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d31b4e973ec1a9..71fbeb7ea432dc 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -365,14 +365,13 @@ def namedtuple(typename, field_names, *, rename=False, module=None): num_fields = len(field_names) arg_list = repr(field_names).replace("'", "")[1:-1] repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' - module_name = f'namedtuple_{typename}' tuple_new = tuple.__new__ _len = len # Create all the named tuple methods to be added to the class namespace s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' - namespace = {'_tuple_new': tuple_new, '__name__': module_name} + namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'} # Note: exec() has the side-effect of interning the typename and field names exec(s, namespace) __new__ = namespace['__new__'] @@ -413,7 +412,6 @@ def __getnewargs__(self): for method in (__new__, _make.__func__, _replace, __repr__, _asdict, __getnewargs__): - method.__module__ = module_name method.__qualname__ = f'{typename}.{method.__name__}' # Helper function used in the class creation From bc6852c014cda6be19d8f39daca1966a9b9d5536 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 21:57:48 -0700 Subject: [PATCH 19/22] Improve variable names and add a micro-optimization for an non-public helper function --- Lib/collections/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 71fbeb7ea432dc..7a10fa59b39dd3 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -416,12 +416,12 @@ def __getnewargs__(self): # Helper function used in the class creation - def reuse_itemgetter(index): + def reuse_itemgetter(index, cache=_nt_itemgetters): try: - return _nt_itemgetters[index] + return cache[index] except KeyError: - getter = _nt_itemgetters[index] = _itemgetter(index) - return getter + itemgetter_object = cache[index] = _itemgetter(index) + return itemgetter_object # Build-up the class namespace dictionary # and use type() to build the result class From c8976744fc3ac2f03c25d942c02271b29cb6352c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 23:53:49 -0700 Subject: [PATCH 20/22] Simplify by in-lining reuse_itemgetter() --- Lib/collections/__init__.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 7a10fa59b39dd3..c6a372bd863e8e 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -414,15 +414,6 @@ def __getnewargs__(self): __repr__, _asdict, __getnewargs__): method.__qualname__ = f'{typename}.{method.__name__}' - # Helper function used in the class creation - - def reuse_itemgetter(index, cache=_nt_itemgetters): - try: - return cache[index] - except KeyError: - itemgetter_object = cache[index] = _itemgetter(index) - return itemgetter_object - # Build-up the class namespace dictionary # and use type() to build the result class class_namespace = { @@ -436,9 +427,14 @@ def reuse_itemgetter(index, cache=_nt_itemgetters): '_asdict': _asdict, '__getnewargs__': __getnewargs__, } + cache = _nt_itemgetters for index, name in enumerate(field_names): doc = _sys.intern(f'Alias for field number {index}') - class_namespace[name] = property(reuse_itemgetter(index), doc=doc) + try: + itemgetter_object = cache[index] + except KeyError: + itemgetter_object = cache[index] = _itemgetter(index) + class_namespace[name] = property(itemgetter_object, doc=doc) result = type(typename, (tuple,), class_namespace) From bd4ea4ece39fd46a953e7c284e4ee6cc4bfc72dd Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 9 Sep 2017 23:56:30 -0700 Subject: [PATCH 21/22] Arrange steps in more logical order --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index c6a372bd863e8e..357e0227c4655b 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -429,11 +429,11 @@ def __getnewargs__(self): } cache = _nt_itemgetters for index, name in enumerate(field_names): - doc = _sys.intern(f'Alias for field number {index}') try: itemgetter_object = cache[index] except KeyError: itemgetter_object = cache[index] = _itemgetter(index) + doc = _sys.intern(f'Alias for field number {index}') class_namespace[name] = property(itemgetter_object, doc=doc) result = type(typename, (tuple,), class_namespace) From ffb78c991e6c2d1719e73ae6b9900fe143f97467 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 10 Sep 2017 08:33:33 -0700 Subject: [PATCH 22/22] Save docstring in local cache instead of interning --- Lib/collections/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 357e0227c4655b..50cf8141731183 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -430,10 +430,11 @@ def __getnewargs__(self): cache = _nt_itemgetters for index, name in enumerate(field_names): try: - itemgetter_object = cache[index] + itemgetter_object, doc = cache[index] except KeyError: - itemgetter_object = cache[index] = _itemgetter(index) - doc = _sys.intern(f'Alias for field number {index}') + itemgetter_object = _itemgetter(index) + doc = f'Alias for field number {index}' + cache[index] = itemgetter_object, doc class_namespace[name] = property(itemgetter_object, doc=doc) result = type(typename, (tuple,), class_namespace)