Add basic chunk format and compression/decompression

Author: foriequal0

Cargo.lock

@@ -10,6 +10,8 @@ hashdb = {path = "../hashdb" }
 codechain-crypto = { git = "https://github.com/CodeChain-io/rust-codechain-crypto.git", version = "0.1" }
 primitives = { git = "https://github.com/CodeChain-io/rust-codechain-primitives.git", version = "0.4" }
 rlp = {path = "../rlp" }
+rlp_derive = { path = "../rlp_derive" }
+snap = "0.2"
 
 [dev-dependencies]
 trie-standardmap = { path = "../trie-standardmap" }

util/merkle/Cargo.toml

@@ -0,0 +1,151 @@
+// Copyright 2019 Kodebox, Inc.
+// This file is part of CodeChain.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use std::io::Error as IoError;
+use std::io::{Cursor, Read, Write};
+
+use primitives::H256;
+use rlp::{DecoderError as RlpDecoderError, RlpStream, UntrustedRlp};
+use snap;
+
+struct ChunkDecompressor<R> {
+    read: R,
+}
+
+impl<R> ChunkDecompressor<R> {
+    fn new(read: R) -> Self {
+        ChunkDecompressor {
+            read,
+        }
+    }
+}
+
+impl<'a> ChunkDecompressor<Cursor<&'a [u8]>> {
+    fn from_slice(slice: &'a [u8]) -> Self {
+        ChunkDecompressor::new(Cursor::new(slice))
+    }
+}
+
+impl<R> ChunkDecompressor<R>
+where
+    R: Read + Clone,
+{
+    fn decompress(self) -> Result<RawChunk, Error> {
+        let mut buf = Vec::new();
+
+        let mut snappy = snap::Reader::new(self.read);
+        snappy.read_to_end(&mut buf)?;
+
+        let rlp = UntrustedRlp::new(&buf);
+        Ok(RawChunk {
+            nodes: rlp.as_list()?,
+        })
+    }
+}
+
+struct ChunkCompressor<W> {
+    write: W,
+}
+
+impl<W> ChunkCompressor<W> {
+    fn new(write: W) -> Self {
+        ChunkCompressor {
+            write,
+        }
+    }
+}
+
+impl<W> ChunkCompressor<W>
+where
+    W: Write,
+{
+    fn compress(self, nodes: &[TerminalNode]) -> Result<(), Error> {
+        let mut rlp = RlpStream::new_list(nodes.len());
+        for node in nodes {
+            rlp.append(node);
+        }
+        let mut snappy = snap::Writer::new(self.write);
+        snappy.write_all(rlp.as_raw())?;
+        Ok(())
+    }
+}
+
+#[test]
+fn test_compress_decompress() {
+    let terminal_nodes = vec![
+        TerminalNode {
+            prefix_path: b"12345".to_vec(),
+            node_rlp: b"45678".to_vec(),
+        },
+        TerminalNode {
+            prefix_path: b"56789".to_vec(),
+            node_rlp: b"123abc".to_vec(),
+        },
+    ];
+
+    let mut buffer = Vec::new();
+    ChunkCompressor::new(&mut buffer).compress(&terminal_nodes).unwrap();
+    let decompressed = ChunkDecompressor::from_slice(&buffer).decompress().unwrap();
+
+    assert_eq!(terminal_nodes, decompressed.nodes);
+}
+
+type Prefix = Vec<u8>;
+
+#[derive(RlpEncodable, RlpDecodable, Eq, PartialEq, Debug)]
+pub struct TerminalNode {
+    pub prefix_path: Prefix,
+    pub node_rlp: Vec<u8>,
+}
+
+pub struct RawChunk {
+    nodes: Vec<TerminalNode>,
+}
+
+struct Metadata {
+    root: H256,
+    prefix: Prefix,
+}
+
+impl RawChunk {
+    fn recover(&self, metadata: Metadata) -> Result<Chunk, ()> {
+        unimplemented!();
+    }
+}
+
+struct Chunk {
+    metadata: Metadata,
+    nodes: Vec<(H256, Vec<u8>)>,
+    unresolved_metadata: Vec<Metadata>,
+}
+
+#[derive(Debug)]
+enum Error {
+    IoError(IoError),
+    RlpDecoderError(RlpDecoderError),
+}
+
+impl From<IoError> for Error {
+    fn from(err: IoError) -> Self {
+        Error::IoError(err)
+    }
+}
+
+impl From<RlpDecoderError> for Error {
+    fn from(err: RlpDecoderError) -> Self {
+        Error::RlpDecoderError(err)
+    }
+}

util/merkle/src/chunk.rs

@@ -21,6 +21,9 @@ extern crate hashdb;
 extern crate memorydb;
 extern crate primitives;
 extern crate rlp;
+#[macro_use]
+extern crate rlp_derive;
+extern crate snap;
 
 #[cfg(test)]
 extern crate trie_standardmap as standardmap;
@@ -31,6 +34,7 @@ use ccrypto::BLAKE_NULL_RLP;
 use hashdb::{DBValue, HashDB};
 use primitives::H256;
 
+pub mod chunk;
 mod nibbleslice;
 pub mod node;
 mod skewed;