Drop writer size hinting/message vec preallocation

In order to avoid significant malloc traffic, messages previously
explicitly stated their serialized length allowing for Vec
preallocation during the message serialization pipeline. This added
some amount of complexity in the serialization code, but did avoid
some realloc() calls.

Instead, here, we drop all the complexity in favor of a fixed 2KiB
buffer for all message serialization. This should not only be
simpler with a similar reduction in realloc() traffic, but also
may reduce heap fragmentation by allocating identically-sized
buffers more often.

Author: TheBlueMatt

fuzz/src/chanmon_consistency.rs

@@ -94,9 +94,6 @@ impl Writer for VecWriter {
 		self.0.extend_from_slice(buf);
 		Ok(())
 	}
-	fn size_hint(&mut self, size: usize) {
-		self.0.reserve_exact(size);
-	}
 }
 
 struct TestChainMonitor {

fuzz/src/chanmon_deser.rs

@@ -18,9 +18,6 @@ impl Writer for VecWriter {
 		self.0.extend_from_slice(buf);
 		Ok(())
 	}
-	fn size_hint(&mut self, size: usize) {
-		self.0.reserve_exact(size);
-	}
 }
 
 #[inline]

fuzz/src/msg_targets/utils.rs

@@ -13,13 +13,9 @@ use lightning::util::ser::Writer;
 pub struct VecWriter(pub Vec<u8>);
 impl Writer for VecWriter {
 	fn write_all(&mut self, buf: &[u8]) -> Result<(), ::std::io::Error> {
-		assert!(self.0.capacity() >= self.0.len() + buf.len());
 		self.0.extend_from_slice(buf);
 		Ok(())
 	}
-	fn size_hint(&mut self, size: usize) {
-		self.0.reserve_exact(size);
-	}
 }
 
 // We attempt to test the strictest behavior we can for a given message, however, some messages
@@ -43,6 +39,7 @@ macro_rules! test_msg {
 				msg.write(&mut w).unwrap();
 
 				assert_eq!(w.0.len(), p);
+				assert_eq!(msg.serialized_length(), p);
 				assert_eq!(&r.into_inner()[..p], &w.0[..p]);
 			}
 		}
@@ -60,6 +57,7 @@ macro_rules! test_msg_simple {
 			if let Ok(msg) = <$MsgType as Readable>::read(&mut r) {
 				let mut w = VecWriter(Vec::new());
 				msg.write(&mut w).unwrap();
+				assert_eq!(msg.serialized_length(), w.0.len());
 
 				let msg = <$MsgType as Readable>::read(&mut ::std::io::Cursor::new(&w.0)).unwrap();
 				let mut w_two = VecWriter(Vec::new());
@@ -82,6 +80,7 @@ macro_rules! test_msg_exact {
 				let mut w = VecWriter(Vec::new());
 				msg.write(&mut w).unwrap();
 				assert_eq!(&r.into_inner()[..], &w.0[..]);
+				assert_eq!(msg.serialized_length(), w.0.len());
 			}
 		}
 	}
@@ -99,6 +98,7 @@ macro_rules! test_msg_hole {
 				let mut w = VecWriter(Vec::new());
 				msg.write(&mut w).unwrap();
 				let p = w.0.len() as usize;
+				assert_eq!(msg.serialized_length(), p);
 
 				assert_eq!(w.0.len(), p);
 				assert_eq!(&r.get_ref()[..$hole], &w.0[..$hole]);

lightning/src/chain/transaction.rs

@@ -75,7 +75,7 @@ impl OutPoint {
 	}
 }
 
-impl_writeable!(OutPoint, 0, { txid, index });
+impl_writeable!(OutPoint, { txid, index });
 
 #[cfg(test)]
 mod tests {

lightning/src/ln/features.rs

@@ -392,7 +392,6 @@ impl InitFeatures {
 	/// Writes all features present up to, and including, 13.
 	pub(crate) fn write_up_to_13<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
 		let len = cmp::min(2, self.flags.len());
-		w.size_hint(len + 2);
 		(len as u16).write(w)?;
 		for i in (0..len).rev() {
 			if i == 0 {
@@ -584,12 +583,6 @@ impl<T: sealed::Context> Features<T> {
 			(byte & unknown_features) != 0
 		})
 	}
-
-	/// The number of bytes required to represent the feature flags present. This does not include
-	/// the length bytes which are included in the serialized form.
-	pub(crate) fn byte_count(&self) -> usize {
-		self.flags.len()
-	}
 }
 
 impl<T: sealed::DataLossProtect> Features<T> {
@@ -702,7 +695,6 @@ impl<T: sealed::ShutdownAnySegwit> Features<T> {
 
 impl<T: sealed::Context> Writeable for Features<T> {
 	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-		w.size_hint(self.flags.len() + 2);
 		(self.flags.len() as u16).write(w)?;
 		for f in self.flags.iter().rev() { // Swap back to big-endian
 			f.write(w)?;
@@ -835,7 +827,7 @@ mod tests {
 	#[test]
 	fn convert_to_context_with_unknown_flags() {
 		// Ensure the `from` context has fewer known feature bytes than the `to` context.
-		assert!(InvoiceFeatures::known().byte_count() < NodeFeatures::known().byte_count());
+		assert!(InvoiceFeatures::known().flags.len() < NodeFeatures::known().flags.len());
 		let invoice_features = InvoiceFeatures::known().set_unknown_feature_optional();
 		assert!(invoice_features.supports_unknown_bits());
 		let node_features: NodeFeatures = invoice_features.to_context();