In-place growth for HashMap.

Author: pczarn

src/libstd/collections/hash/map.rs

@@ -23,7 +23,7 @@ use iter::{self, Iterator, ExactSizeIterator, IntoIterator, IteratorExt, FromIte
 use marker::Sized;
 use mem::{self, replace};
 use num::{Int, UnsignedInt};
-use ops::{Drop, FnMut, Index, IndexMut};
+use ops::{Deref, DerefMut, Drop, FnMut, Index, IndexMut};
 use option::Option::{self, Some, None};
 use rand::{self, Rng};
 use result::Result::{self, Ok, Err};
@@ -661,12 +661,20 @@ impl<K, V, S, H> HashMap<K, V, S>
         }
 
         // Grow the table.
-        let mut destination = RawTable::new(new_capacity);
+        let is_inplace = self.table.grow_inplace(new_capacity);
+
+        let mut destination = if is_inplace {
+            // Resizing in-place.
+            None
+        } else {
+            // Borrow self.table in both branches to satisfy the checker.
+            Some(RawTable::new(new_capacity))
+        };
 
         // Iterate over `old_capacity` buckets, which constitute half of
         // the table which was resized in-place, or the entire
         // `old_table`.
-        let mut bucket = Bucket::at_index(&mut self.table, 0).ok().unwrap();
+        let mut bucket = Bucket::at_index(&mut self.table, 0).ok().unwrap().iter_to(old_capacity);
 
         // "So a few of the first shall be last: for many be called,
         // but few chosen."
@@ -717,15 +725,23 @@ impl<K, V, S, H> HashMap<K, V, S>
                 Full(bucket) => {
                     let h = *bucket.read().0;
                     let (b, k, v) = bucket.take();
-                    insert_hashed_ordered(&mut destination, h, k, v);
-                    b.into_bucket()
+
+                    if let Some(ref mut dest) = destination {
+                        insert_hashed_ordered(dest, h, k, v);
+                        b.into_bucket()
+                    } else {
+                        // Resizing in-place.
+                        insert_hashed_ordered(b.into_bucket(), h, k, v)
+                    }
                 }
                 Empty(b) => b.into_bucket()
             };
             bucket.next(); // wraps at old_capacity
         }
 
-        replace(bucket.into_table(), destination);
+        if let Some(dest) = destination {
+            replace(bucket.into_table(), dest);
+        }
     }
 
     /// Shrinks the capacity of the map as much as possible. It will drop

src/libstd/collections/hash/table.rs

@@ -22,9 +22,9 @@ use mem::{self, min_align_of, size_of};
 use num::{Int, UnsignedInt};
 use ops::{Deref, DerefMut, Drop};
 use option::Option::{self, Some, None};
-use ptr::{self, Unique, PtrExt, copy_nonoverlapping_memory, zero_memory};
+use ptr::{self, Unique, PtrExt, copy_memory, copy_nonoverlapping_memory, zero_memory};
 use result::Result::{self, Ok, Err};
-use rt::heap::{allocate, deallocate};
+use rt::heap::{allocate, deallocate, reallocate_inplace};
 use collections::hash_state::HashState;
 use core::nonzero::NonZero;
 
@@ -277,6 +277,14 @@ impl<K, V, M> Bucket<K, V, M> where RawTable<K, V>: BorrowFrom<M> {
         Bucket::at_index(table, 0).map(|b| b.into_iter()).unwrap_or_else(|b| b.into_iter())
     }
 
+    /// Narrows down the range of iteration, which must be a power of 2.
+    pub fn iter_to(mut self, limit: usize) -> Bucket<K, V, M> {
+        assert!(limit <= self.0.table.capacity());
+        assert!(limit.is_power_of_two());
+        self.0.capacity = limit;
+        self
+    }
+
     /// Reads a bucket at a given index, returning an enum indicating whether
     /// it's initialized or not. You need to match on this enum to get
     /// the appropriate types to call most of the other functions in
@@ -521,6 +529,47 @@ impl<K, V> RawTable<K, V> {
         }
     }
 
+    pub fn grow_inplace(&mut self, capacity: uint) -> bool {
+        assert!(capacity.is_power_of_two());
+        assert!(capacity >= self.capacity);
+
+        if self.middle.ptr.is_null() {
+            return false;
+        }
+
+        let new_size = checked_size_generic::<K, V>(capacity);
+
+        unsafe {
+            let ptr = self.middle.ptr.offset(-(self.capacity as isize)) as *mut u8;
+            let is_inplace = reallocate_inplace(ptr,
+                                                size_generic::<K, V>(self.capacity),
+                                                new_size,
+                                                align::<K, V>()) >= new_size;
+
+            if is_inplace {
+                let cap_diff = (capacity - self.capacity) as isize;
+                let hashes = self.middle.ptr.offset(cap_diff) as *mut Option<SafeHash>;
+                // Copy the array of hashes. Maybe it's already in cache.
+                if size_of::<(K, V)>() >= size_of::<Option<SafeHash>>() {
+                    // The regions of memory occupied by old and new hash arrays are disjoint.
+                    // before: [KVKVKVKV|h h h h ]
+                    // after:  [KVKVKVKV|KVKVKVKV|h h h h h h h h ]
+                    copy_nonoverlapping_memory(hashes, self.middle.ptr as *const _, self.capacity);
+                } else {
+                    // before: [KVKVKVKV|h   h   |h   h   ]
+                    // after:  [KVKVKVKV|KVKVKVKV|h   h   h   h   h   h   h   h   ]
+                    copy_memory(hashes, self.middle.ptr as *const _, self.capacity);
+                }
+                zero_memory(hashes.offset(self.capacity as int), capacity - self.capacity);
+
+                self.middle = Unique(self.middle.ptr.offset(cap_diff));
+                self.capacity = capacity;
+            }
+
+            return is_inplace;
+        }
+    }
+
     /// The hashtable's capacity, similar to a vector's.
     pub fn capacity(&self) -> usize {
         self.capacity