From e0234cf08aa26aaedb60941391267f98b501b193 Mon Sep 17 00:00:00 2001 From: Geunwoo Kim Date: Tue, 24 Jul 2018 17:29:40 +0900 Subject: [PATCH 1/2] Add triehash module and edit test case in triedbmut to use triehash --- util/merkle/src/lib.rs | 1 + util/merkle/src/triedbmut.rs | 317 ++++++++++++++++++++++++++++------- util/merkle/src/triehash.rs | 249 +++++++++++++++++++++++++++ 3 files changed, 504 insertions(+), 63 deletions(-) create mode 100644 util/merkle/src/triehash.rs diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index 6002ac7586..6cd6962600 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -34,6 +34,7 @@ mod nibbleslice; pub mod node; mod skewed; pub mod triedbmut; +pub mod triehash; pub use skewed::skewed_merkle_root; diff --git a/util/merkle/src/triedbmut.rs b/util/merkle/src/triedbmut.rs index d38c3c8449..6f22d21e8a 100644 --- a/util/merkle/src/triedbmut.rs +++ b/util/merkle/src/triedbmut.rs @@ -364,14 +364,77 @@ impl<'a> TrieMut for TrieDBMut<'a> { } } + #[cfg(test)] mod tests { + use super::super::triehash::trie_root; use super::super::TrieMut; use super::*; use ccrypto::BLAKE_NULL_RLP; use memorydb::*; + use primitives::bytes::ToPretty; use standardmap::*; + fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &[(Vec, Vec)]) -> TrieDBMut<'db> { + let mut t = TrieDBMut::new(db, root); + for i in 0..v.len() { + let key: &[u8] = &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(key, val).unwrap(); + } + t + } + + fn unpopulate_trie<'db>(t: &mut TrieDBMut<'db>, v: &[(Vec, Vec)]) { + for i in v { + let key: &[u8] = &i.0; + t.remove(key).unwrap(); + } + } + + #[test] + fn playpen() { + let mut seed = H256::new(); + for test_i in 0..10 { + if test_i % 50 == 0 { + println!("{:?} of 10000 stress tests done", test_i); + } + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 100, + }.make_with(&mut seed); + + let real = trie_root(x.clone()); + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + + if *memtrie.root() != real { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), real); + unpopulate_trie(&mut memtrie, &x); + + if *memtrie.root() != BLAKE_NULL_RLP { + println!("- TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), BLAKE_NULL_RLP); + } + } + #[test] fn init() { let mut memdb = MemoryDB::new(); @@ -386,42 +449,25 @@ mod tests { let mut root = H256::new(); let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); - - let key = blake256(&[0x01u8, 0x23]); - let slice = NibbleSlice::new(&key); - let node = RlpNode::Leaf(slice, DBValue::from_slice(&[0x01u8, 0x23])); - let node_rlp = RlpNode::encoded(node); - let hash = blake256(&node_rlp); - - assert_eq!(*t.root(), hash); + assert_eq!(*t.root(), trie_root(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); } #[test] fn remove_to_empty() { - let big_value = b"0000000000000000000000000000000"; - let big_value1 = b"1111111111111111111111111111111"; - let big_value2 = b"2222222222222222222222222222222"; + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t1 = TrieDBMut::new(&mut memdb, &mut root); + t1.insert(&[0x01, 0x23], big_value).unwrap(); + t1.insert(&[0x01, 0x34], big_value).unwrap(); let mut memdb2 = MemoryDB::new(); let mut root2 = H256::new(); let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); - t2.insert(&[0x01], big_value).unwrap(); - t2.insert(&[0x01, 0x23], big_value1).unwrap(); - t2.insert(&[0x01, 0x34], big_value2).unwrap(); - - assert_eq!(t2.get(&[0x01]).unwrap().unwrap(), DBValue::from_slice(big_value)); - assert_eq!(t2.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(big_value1)); - assert_eq!(t2.get(&[0x01, 0x34]).unwrap().unwrap(), DBValue::from_slice(big_value2)); // Insert split leaf - - assert_eq!(t2.contains(&[0x01]).unwrap(), true); - t2.remove(&[0x01]).unwrap().unwrap(); - assert_eq!(t2.contains(&[0x01]).unwrap(), false); - assert_eq!(t2.contains(&[0x01, 0x34]).unwrap(), true); - t2.remove(&[0x01, 0x34]).unwrap().unwrap(); // Remove Leaf which is followed by changing branch to leaf - assert_eq!(t2.contains(&[0x01, 0x34]).unwrap(), false); - assert_eq!(t2.contains(&[0x01, 0x23]).unwrap(), true); - t2.remove(&[0x01, 0x23]).unwrap(); - assert_eq!(t2.contains(&[0x01, 0x23]).unwrap(), false); + t2.insert(&[0x01, 0x23], big_value).unwrap(); + t2.insert(&[0x01, 0x34], big_value).unwrap(); + t2.remove(&[0x01]).unwrap(); } #[test] @@ -431,8 +477,79 @@ mod tests { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![(vec![0x01u8, 0x23], vec![0x23u8, 0x45])])); + } - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x23u8, 0x45])) + #[test] + fn insert_make_branch_root() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x11u8, 0x23], vec![0x11u8, 0x23])]) + ); + } + + #[test] + fn insert_into_branch_root() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]) + ); + } + + #[test] + fn insert_value_into_branch_root() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[], &[0x0]).unwrap(); + assert_eq!(*t.root(), trie_root(vec![(vec![], vec![0x0]), (vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); + } + + #[test] + fn insert_split_leaf() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![(vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x01u8, 0x34], vec![0x01u8, 0x34])]) + ); + } + + #[test] + fn insert_split_extenstion() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01, 0x23, 0x45], &[0x01]).unwrap(); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]).unwrap(); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ]) + ); } #[test] @@ -445,9 +562,25 @@ mod tests { let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0).unwrap(); t.insert(&[0x11u8, 0x23], big_value1).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![(vec![0x01u8, 0x23], big_value0.to_vec()), (vec![0x11u8, 0x23], big_value1.to_vec())]) + ); + } - assert_eq!(t.get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(big_value0)); - assert_eq!(t.get(&[0x11u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(big_value1)); + #[test] + fn insert_duplicate_value() { + let big_value = b"00000000000000000000000000000000"; + + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], big_value).unwrap(); + t.insert(&[0x11u8, 0x23], big_value).unwrap(); + assert_eq!( + *t.root(), + trie_root(vec![(vec![0x01u8, 0x23], big_value.to_vec()), (vec![0x11u8, 0x23], big_value.to_vec())]) + ); } #[test] @@ -458,6 +591,74 @@ mod tests { assert_eq!(t.get(&[0x5]), Ok(None)); } + #[test] + fn test_at_one() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); + + assert_eq!(t.get(&[0x1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x1u8, 0x23])); + } + + #[test] + fn test_at_three() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]).unwrap(); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]).unwrap(); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]).unwrap(); + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); + assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + + assert_eq!(t.get(&[0x01, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0xf1u8, 0x23])); + assert_eq!(t.get(&[0x81, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x81u8, 0x23])); + assert_eq!(t.get(&[0x82, 0x23]), Ok(None)); + } + + #[test] + fn stress() { + let mut seed = H256::new(); + for _ in 0..50 { + let x = StandardMap { + alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), + min_key: 5, + journal_key: 0, + value_mode: ValueMode::Index, + count: 4, + }.make_with(&mut seed); + + let real = trie_root(x.clone()); + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); + let mut y = x.clone(); + y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); + let mut memdb2 = MemoryDB::new(); + let mut root2 = H256::new(); + let mut memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); + if *memtrie.root() != real || *memtrie_sorted.root() != real { + println!("TRIE MISMATCH"); + println!(""); + println!("ORIGINAL... {:?}", memtrie.root()); + for i in &x { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("SORTED... {:?}", memtrie_sorted.root()); + for i in &y { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + } + assert_eq!(*memtrie.root(), real); + assert_eq!(*memtrie_sorted.root(), real); + } + } + #[test] fn test_trie_existing() { let mut root = H256::new(); @@ -468,48 +669,35 @@ mod tests { } { - let t = TrieDBMut::from_existing(&mut db, &mut root); // Why can't I use '?' behind this evaluation - assert_eq!(t.unwrap().get(&[0x01u8, 0x23]).unwrap().unwrap(), DBValue::from_slice(&[0x01u8, 0x23])); + let _ = TrieDBMut::from_existing(&mut db, &mut root); } } #[test] - #[ignore] - fn insert_empty() { - let mut seed = H256::new(); - let x = StandardMap { - alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), - min_key: 5, - journal_key: 0, - value_mode: ValueMode::Index, - count: 4, - }.make_with(&mut seed); - + fn from_null_rlp_succeeds() { + let mut root = BLAKE_NULL_RLP; let mut db = MemoryDB::new(); - let mut root = H256::new(); - let mut t = TrieDBMut::new(&mut db, &mut root); - for &(ref key, ref value) in &x { - t.insert(key, value).unwrap(); - } - - - for &(ref key, _) in &x { - t.insert(key, &[]).unwrap(); - } + TrieDBMut::from_existing(&mut db, &mut root).unwrap(); + } - assert!(t.is_empty()); - assert_eq!(*t.root(), BLAKE_NULL_RLP); + #[test] + #[should_panic] + fn from_zero_fails() { + let mut root = H256::zero(); + let mut db = MemoryDB::new(); + TrieDBMut::from_existing(&mut db, &mut root).unwrap(); } #[test] - fn random_insert_remove() { + #[ignore] + fn insert_empty() { let mut seed = H256::new(); let x = StandardMap { alphabet: Alphabet::Custom(b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_".to_vec()), min_key: 5, journal_key: 0, value_mode: ValueMode::Index, - count: 30, + count: 4, }.make_with(&mut seed); let mut db = MemoryDB::new(); @@ -519,8 +707,10 @@ mod tests { t.insert(key, value).unwrap(); } + assert_eq!(*t.root(), trie_root(x.clone())); + for &(ref key, _) in &x { - t.remove(key).unwrap(); + t.insert(key, &[]).unwrap(); } assert!(t.is_empty()); @@ -542,13 +732,14 @@ mod tests { let mut root = H256::new(); let mut t = TrieDBMut::new(&mut db, &mut root); for &(ref key, ref value) in &x { - assert_eq!(Ok(None), t.insert(key, value)); - assert_eq!(Ok(Some(DBValue::from_slice(value))), t.insert(key, value)); + assert!(t.insert(key, value).unwrap().is_none()); + assert_eq!(t.insert(key, value).unwrap(), Some(DBValue::from_slice(value))); } for (key, value) in x { - assert_eq!(Ok(Some(DBValue::from_slice(&value))), t.remove(&key)); - assert_eq!(Ok(None), t.remove(&key)); + assert_eq!(t.remove(&key).unwrap(), Some(DBValue::from_slice(&value))); + assert!(t.remove(&key).unwrap().is_none()); } } } + diff --git a/util/merkle/src/triehash.rs b/util/merkle/src/triehash.rs new file mode 100644 index 0000000000..e515fbbfde --- /dev/null +++ b/util/merkle/src/triehash.rs @@ -0,0 +1,249 @@ +// Copyright 2018 Kodebox, Inc. +// This file is part of CodeChain. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Generetes trie root. +//! +//! This module should be used to generate trie root hash. + +use std::cmp; +use std::collections::BTreeMap; + +use ccrypto::blake256; +use primitives::H256; +use rlp::RlpStream; + +fn shared_prefix_len(first: &[T], second: &[T]) -> usize { + let len = cmp::min(first.len(), second.len()); + (0..len).take_while(|&i| first[i] == second[i]).count() +} + +/// Generates a trie root hash for a vector of key-values +pub fn trie_root(input: I) -> H256 +where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, { + // Make key into hash value, which is blake256(key) + let gen_input: Vec<_> = input.into_iter().map(|(k, v)| (blake256(k), v)).collect(); + let gen_input: Vec<_> = gen_input + // first put elements into btree to sort them and to remove duplicates + .into_iter() + .collect::>() + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(k.as_ref()), v) ) + .collect(); + + gen_trie_root(&gen_input) +} + +fn gen_trie_root, B: AsRef<[u8]>>(input: &[(A, B)]) -> H256 { + let mut stream = RlpStream::new(); + hash256rlp(input, 0, &mut stream); + blake256(stream.out()) +} + +/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 +/// +/// Input values are in range `[0, 0xf]`. +/// +/// ```markdown +/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +/// [1,2,3,4,5] 0x112345 // 5 > 3 +/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +/// [0,1,2,3,4] 0x101234 // 5 > 3 +/// [1,2,3,4] 0x001234 // 4 > 3 +/// ``` +fn hex_prefix_encode(nibbles: &[u8]) -> Vec { + let inlen = nibbles.len(); + let oddness_factor = inlen % 2; + // next even number divided by two + let reslen = (inlen + 2) >> 1; + let mut res = Vec::with_capacity(reslen); + + let first_byte = { + let mut bits = (oddness_factor as u8) << 4; + if oddness_factor == 1 { + bits += nibbles[0]; + } + bits + }; + + res.push(first_byte); + + let mut offset = oddness_factor; + while offset < inlen { + let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; + res.push(byte); + offset += 2; + } + + res +} + +/// Converts slice of bytes to nibbles. +fn as_nibbles(bytes: &[u8]) -> Vec { + let mut res = Vec::with_capacity(bytes.len() * 2); + for i in 0..bytes.len() { + let byte = bytes[i]; + res.push(byte >> 4); + res.push(byte & 0b1111); + } + res +} + +fn hash256rlp, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, stream: &mut RlpStream) { + let inlen = input.len(); + + // in case of empty slice, just append empty data + if inlen == 0 { + stream.append_empty_data(); + return + } + + // take slices + let key: &[u8] = &input[0].0.as_ref(); + let value: &[u8] = &input[0].1.as_ref(); + + // if the slice contains just one item, append the suffix of the key + // and then append value + if inlen == 1 { + stream.begin_list(2); + stream.append(&hex_prefix_encode(&key[pre_len..])); + stream.append(&value); + return + } + + // get length of the longest shared prefix in slice keys + let shared_prefix = input.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(key.len(), | acc, &(ref k, _) | { + cmp::min(shared_prefix_len(key, k.as_ref()), acc) + }); + + // an item for every possible nibble/suffix + // + 1 for data + stream.begin_list(17); + + // Append partial path as a first element of branch + println!("{} and {}", pre_len, shared_prefix); + stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix])); + + let mut begin: usize = 0; + + // iterate over all possible nibbles + for i in 0..16 { + // count how many successive elements have same next nibble + let len = match begin < input.len() { + true => input[begin..].iter().take_while(|pair| pair.0.as_ref()[shared_prefix] == i).count(), + false => 0, + }; + + // if at least 1 successive element has the same nibble + // append their suffixes + match len { + 0 => { + stream.append_empty_data(); + } + _ => hash256aux(&input[begin..(begin + len)], shared_prefix + 1, stream), + } + begin += len; + } +} + +fn hash256aux, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, stream: &mut RlpStream) { + let mut s = RlpStream::new(); + hash256rlp(input, pre_len, &mut s); + let out = s.out(); + + stream.append(&blake256(out)); +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_nibbles() { + let v = vec![0x31, 0x23, 0x45]; + let e = vec![3, 1, 2, 3, 4, 5]; + assert_eq!(as_nibbles(&v), e); + + // A => 65 => 0x41 => [4, 1] + let v: Vec = From::from("A"); + let e = vec![4, 1]; + assert_eq!(as_nibbles(&v), e); + } + + #[test] + fn test_hex_prefix_encode() { + let v = vec![0, 0, 1, 2, 3, 4, 5]; + let e = vec![0x10, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x00, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4]; + let e = vec![0x00, 0x12, 0x34]; + let h = hex_prefix_encode(&v); + assert_eq!(h, e); + } + + #[test] + fn test_triehash_out_of_order() { + assert_eq!( + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]), + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + ]) + ); + } + + #[test] + fn test_shared_prefix() { + let a = vec![1, 2, 3, 4, 5, 6]; + let b = vec![4, 2, 3, 4, 5, 6]; + assert_eq!(shared_prefix_len(&a, &b), 0); + } + + #[test] + fn test_shared_prefix2() { + let a = vec![1, 2, 3, 3, 5]; + let b = vec![1, 2, 3]; + assert_eq!(shared_prefix_len(&a, &b), 3); + } + + #[test] + fn test_shared_prefix3() { + let a = vec![1, 2, 3, 4, 5, 6]; + let b = vec![1, 2, 3, 4, 5, 6]; + assert_eq!(shared_prefix_len(&a, &b), 6); + } +} From d6a3974e9e50459d7a424dcbecb2d5796d86bf34 Mon Sep 17 00:00:00 2001 From: Geunwoo Kim Date: Tue, 24 Jul 2018 23:54:06 +0900 Subject: [PATCH 2/2] Add Triedb module --- util/merkle/src/lib.rs | 22 ++++++ util/merkle/src/triedb.rs | 138 +++++++++++++++++++++++++++++++++++ util/merkle/src/triedbmut.rs | 39 ++-------- util/merkle/src/triehash.rs | 1 - 4 files changed, 165 insertions(+), 35 deletions(-) create mode 100644 util/merkle/src/triedb.rs diff --git a/util/merkle/src/lib.rs b/util/merkle/src/lib.rs index 6cd6962600..79c7dc0aac 100644 --- a/util/merkle/src/lib.rs +++ b/util/merkle/src/lib.rs @@ -27,16 +27,20 @@ extern crate trie_standardmap as standardmap; use std::fmt; +use ccrypto::BLAKE_NULL_RLP; use hashdb::DBValue; use primitives::H256; mod nibbleslice; pub mod node; mod skewed; +pub mod triedb; pub mod triedbmut; pub mod triehash; pub use skewed::skewed_merkle_root; +pub use triedb::TrieDB; +pub use triedbmut::TrieDBMut; /// Trie Errors. /// @@ -62,6 +66,24 @@ impl fmt::Display for TrieError { /// Trie result type. Boxed to avoid copying around extra space for `H256`s on successful queries. pub type Result = ::std::result::Result>; +/// A key-value datastore implemented as a database-backed modified Merkle tree. +pub trait Trie { + /// Return the root of the trie. + fn root(&self) -> &H256; + + /// Is the trie empty? + fn is_empty(&self) -> bool { + *self.root() == BLAKE_NULL_RLP + } + + /// Does the trie contain a given key? + fn contains(&self, key: &[u8]) -> Result { + self.get(key).map(|x| x.is_some()) + } + + /// What is the value of the given key in this trie? + fn get(&self, key: &[u8]) -> Result>; +} /// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait TrieMut { diff --git a/util/merkle/src/triedb.rs b/util/merkle/src/triedb.rs new file mode 100644 index 0000000000..a6178b27f2 --- /dev/null +++ b/util/merkle/src/triedb.rs @@ -0,0 +1,138 @@ +// Copyright 2018 Kodebox, Inc. +// This file is part of CodeChain. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use ccrypto::blake256; +use hashdb::DBValue; +use hashdb::HashDB; +use primitives::H256; + +use super::nibbleslice::NibbleSlice; +use super::node::Node as RlpNode; +use super::{Trie, TrieError}; +/// A `Trie` implementation using a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object. +/// Use `get` and `contains` to query values associated with keys in the trie. +/// +/// # Example +/// ``` +/// extern crate hashdb; +/// extern crate memorydb; +/// extern crate primitives; +/// extern crate codechain_merkle as cmerkle; +/// +/// use cmerkle::*; +/// use hashdb::*; +/// use memorydb::*; +/// use primitives::H256; +/// +/// fn main() { +/// let mut memdb = MemoryDB::new(); +/// let mut root = H256::new(); +/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar").unwrap(); +/// let t = TrieDB::new(&memdb, &root).unwrap(); +/// assert!(t.contains(b"foo").unwrap()); +/// assert_eq!(t.get(b"foo").unwrap().unwrap(), DBValue::from_slice(b"bar")); +/// } +/// ``` +pub struct TrieDB<'db> { + db: &'db HashDB, + root: &'db H256, +} + +impl<'db> TrieDB<'db> { + /// Create a new trie with the backing database `db` and `root` + /// Returns an error if `root` does not exist + pub fn new(db: &'db HashDB, root: &'db H256) -> super::Result { + if !db.contains(root) { + Err(Box::new(TrieError::InvalidStateRoot(*root))) + } else { + Ok(TrieDB { + db, + root, + }) + } + } + + /// Get the backing database. + pub fn db(&'db self) -> &'db HashDB { + self.db + } + + /// Get auxiliary + fn get_aux(&self, path: NibbleSlice, cur_node_hash: Option) -> super::Result> { + match cur_node_hash { + Some(hash) => { + let node_rlp = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; + + match RlpNode::decoded(&node_rlp) { + Some(RlpNode::Leaf(partial, value)) => { + if partial == path { + Ok(Some(value)) + } else { + Ok(None) + } + } + Some(RlpNode::Branch(partial, children)) => { + if path.starts_with(&partial) { + self.get_aux(path.mid(partial.len() + 1), children[path.mid(partial.len()).at(0) as usize]) + } else { + Ok(None) + } + } + None => Ok(None), + } + } + None => Ok(None), + } + } +} + +impl<'db> Trie for TrieDB<'db> { + fn root(&self) -> &H256 { + self.root + } + + fn get(&self, key: &[u8]) -> super::Result> { + let path = blake256(key); + let root = *self.root; + + self.get_aux(NibbleSlice::new(&path), Some(root)) + } +} + +#[cfg(test)] +mod tests { + use super::super::*; + use super::*; + use memorydb::*; + + #[test] + fn get() { + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + { + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(b"A", b"ABC").unwrap(); + t.insert(b"B", b"ABCBA").unwrap(); + } + + let t = TrieDB::new(&memdb, &root).unwrap(); + assert_eq!(t.get(b"A"), Ok(Some(DBValue::from_slice(b"ABC")))); + assert_eq!(t.get(b"B"), Ok(Some(DBValue::from_slice(b"ABCBA")))); + assert_eq!(t.get(b"C"), Ok(None)); + } +} diff --git a/util/merkle/src/triedbmut.rs b/util/merkle/src/triedbmut.rs index 6f22d21e8a..131d358f70 100644 --- a/util/merkle/src/triedbmut.rs +++ b/util/merkle/src/triedbmut.rs @@ -20,9 +20,10 @@ use hashdb::DBValue; use hashdb::HashDB; use primitives::H256; -use super::node::Node as RlpNode; -use super::{TrieError, TrieMut}; +use super::{Trie, TrieError, TrieMut}; use nibbleslice::NibbleSlice; +use node::Node as RlpNode; +use triedb::TrieDB; fn empty_children() -> [Option; 16] { @@ -170,34 +171,6 @@ impl<'a> TrieDBMut<'a> { } } - /// Get auxiliary - fn get_aux(&self, path: NibbleSlice, cur_node_hash: Option) -> super::Result> { - match cur_node_hash { - Some(hash) => { - let node_rlp = self.db.get(&hash).ok_or_else(|| Box::new(TrieError::IncompleteDatabase(hash)))?; - - match RlpNode::decoded(&node_rlp) { - Some(RlpNode::Leaf(partial, value)) => { - if partial == path { - Ok(Some(value)) - } else { - Ok(None) - } - } - Some(RlpNode::Branch(partial, children)) => { - if path.starts_with(&partial) { - self.get_aux(path.mid(partial.len() + 1), children[path.mid(partial.len()).at(0) as usize]) - } else { - Ok(None) - } - } - None => Ok(None), - } - } - None => Ok(None), - } - } - /// Remove auxiliary fn remove_aux( &mut self, @@ -334,10 +307,9 @@ impl<'a> TrieMut for TrieDBMut<'a> { } fn get(&self, key: &[u8]) -> super::Result> { - let path = blake256(key); - let cur_hash = *self.root; + let t = TrieDB::new(self.db, self.root)?; - self.get_aux(NibbleSlice::new(&path), Some(cur_hash)) + t.get(key) } fn insert(&mut self, key: &[u8], value: &[u8]) -> super::Result> { @@ -742,4 +714,3 @@ mod tests { } } } - diff --git a/util/merkle/src/triehash.rs b/util/merkle/src/triehash.rs index e515fbbfde..d975a7745a 100644 --- a/util/merkle/src/triehash.rs +++ b/util/merkle/src/triehash.rs @@ -142,7 +142,6 @@ fn hash256rlp, B: AsRef<[u8]>>(input: &[(A, B)], pre_len: usize, stream.begin_list(17); // Append partial path as a first element of branch - println!("{} and {}", pre_len, shared_prefix); stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix])); let mut begin: usize = 0;