Add basic chunk format and compression/decompression

Author: foriequal0

Cargo.lock

@@ -22,6 +22,7 @@ parking_lot = "0.6.0"
 primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" }
 rand = "0.6.1"
 rlp = { path = "../util/rlp" }
+rlp_derive = { path = "../util/rlp_derive" }
 snap = "0.2"
 time = "0.1"
 token-generator = "0.1.0"

sync/Cargo.toml

@@ -37,6 +37,8 @@ extern crate never_type;
 extern crate primitives;
 extern crate rand;
 extern crate rlp;
+#[macro_use]
+extern crate rlp_derive;
 extern crate snap;
 #[cfg(test)]
 extern crate tempfile;
@@ -47,6 +49,7 @@ extern crate trie_standardmap;
 extern crate util_error;
 
 mod block;
+mod snapshot;
 mod transaction;
 
 pub use crate::block::{BlockSyncEvent, BlockSyncExtension, BlockSyncSender};

sync/src/lib.rs

@@ -0,0 +1,131 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use std::io::Error as IoError;
+use std::io::{Cursor, Read, Write};
+
+use rlp::{DecoderError as RlpDecoderError, RlpStream, UntrustedRlp};
+use snap;
+
+struct ChunkDecompressor<R> {
+    read: R,
+}
+
+impl<R> ChunkDecompressor<R> {
+    fn new(read: R) -> Self {
+        ChunkDecompressor {
+            read,
+        }
+    }
+}
+
+impl<'a> ChunkDecompressor<Cursor<&'a [u8]>> {
+    fn from_slice(slice: &'a [u8]) -> Self {
+        ChunkDecompressor::new(Cursor::new(slice))
+    }
+}
+
+impl<R> ChunkDecompressor<R>
+where
+    R: Read + Clone,
+{
+    fn decompress(self) -> Result<RawChunk, Error> {
+        let mut buf = Vec::new();
+
+        let mut snappy = snap::Reader::new(self.read);
+        snappy.read_to_end(&mut buf)?;
+
+        let rlp = UntrustedRlp::new(&buf);
+        Ok(RawChunk {
+            nodes: rlp.as_list()?,
+        })
+    }
+}
+
+struct ChunkCompressor<W> {
+    write: W,
+}
+
+impl<W> ChunkCompressor<W> {
+    fn new(write: W) -> Self {
+        ChunkCompressor {
+            write,
+        }
+    }
+}
+
+impl<W> ChunkCompressor<W>
+where
+    W: Write,
+{
+    fn compress(self, nodes: &[TerminalNode]) -> Result<(), Error> {
+        let mut rlp = RlpStream::new_list(nodes.len());
+        for node in nodes {
+            rlp.append(node);
+        }
+        let mut snappy = snap::Writer::new(self.write);
+        snappy.write_all(rlp.as_raw())?;
+        Ok(())
+    }
+}
+
+#[test]
+fn test_compress_decompress() {
+    let terminal_nodes = vec![
+        TerminalNode {
+            prefix_path: b"12345".to_vec(),
+            node_rlp: b"45678".to_vec(),
+        },
+        TerminalNode {
+            prefix_path: b"56789".to_vec(),
+            node_rlp: b"123abc".to_vec(),
+        },
+    ];
+
+    let mut buffer = Vec::new();
+    ChunkCompressor::new(&mut buffer).compress(&terminal_nodes).unwrap();
+    let decompressed = ChunkDecompressor::from_slice(&buffer).decompress().unwrap();
+
+    assert_eq!(terminal_nodes, decompressed.nodes);
+}
+
+#[derive(RlpEncodable, RlpDecodable, Eq, PartialEq, Debug)]
+struct TerminalNode {
+    prefix_path: Vec<u8>,
+    node_rlp: Vec<u8>,
+}
+
+struct RawChunk {
+    nodes: Vec<TerminalNode>,
+}
+
+#[derive(Debug)]
+enum Error {
+    IoError(IoError),
+    RlpDecoderError(RlpDecoderError),
+}
+
+impl From<IoError> for Error {
+    fn from(err: IoError) -> Self {
+        Error::IoError(err)
+    }
+}
+
+impl From<RlpDecoderError> for Error {
+    fn from(err: RlpDecoderError) -> Self {
+        Error::RlpDecoderError(err)
+    }
+}