Trie-based execution proof (#177)

* TrieBasedBackend

* trie tests

* redunant return_value removed

* use Trie::get_with to record trie proofs

Author: svyatonik

Cargo.lock

@@ -17,13 +17,11 @@
 //! Client backend that uses RocksDB database as storage. State is still kept in memory.
 
 extern crate substrate_client as client;
-extern crate ethereum_types;
 extern crate kvdb_rocksdb;
 extern crate kvdb;
 extern crate hashdb;
 extern crate memorydb;
 extern crate parking_lot;
-extern crate patricia_trie;
 extern crate substrate_state_machine as state_machine;
 extern crate substrate_primitives as primitives;
 extern crate substrate_runtime_support as runtime_support;
@@ -38,23 +36,23 @@ extern crate kvdb_memorydb;
 
 use std::sync::Arc;
 use std::path::PathBuf;
-use std::collections::HashMap;
 
 use codec::Slicable;
-use ethereum_types::H256 as TrieH256;
-use hashdb::{DBValue, HashDB};
+use hashdb::DBValue;
 use kvdb_rocksdb::{Database, DatabaseConfig};
 use kvdb::{KeyValueDB, DBTransaction};
 use memorydb::MemoryDB;
 use parking_lot::RwLock;
-use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::bft::Justification;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hashing, HashingFor, Zero};
 use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
 use state_machine::CodeExecutor;
 
+/// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
+pub type DbState = state_machine::TrieBackend;
+
 /// Database settings.
 pub struct DatabaseSettings {
 	/// Cache size in bytes. If `None` default is used.
@@ -301,133 +299,6 @@ impl<Block: BlockT> client::backend::BlockImportOperation<Block> for BlockImport
 	}
 }
 
-struct Ephemeral<'a> {
-	backing: &'a KeyValueDB,
-	overlay: &'a mut MemoryDB,
-}
-
-impl<'a> HashDB for Ephemeral<'a> {
-	fn keys(&self) -> HashMap<TrieH256, i32> {
-		self.overlay.keys() // TODO: iterate backing
-	}
-
-	fn get(&self, key: &TrieH256) -> Option<DBValue> {
-		match self.overlay.raw(key) {
-			Some((val, i)) => {
-				if i <= 0 {
-					None
-				} else {
-					Some(val)
-				}
-			}
-			None => {
-				match self.backing.get(::columns::STATE, &key.0[..]) {
-					Ok(x) => x,
-					Err(e) => {
-						warn!("Failed to read from DB: {}", e);
-						None
-					}
-				}
-			}
-		}
-	}
-
-	fn contains(&self, key: &TrieH256) -> bool {
-		self.get(key).is_some()
-	}
-
-	fn insert(&mut self, value: &[u8]) -> TrieH256 {
-		self.overlay.insert(value)
-	}
-
-	fn emplace(&mut self, key: TrieH256, value: DBValue) {
-		self.overlay.emplace(key, value)
-	}
-
-	fn remove(&mut self, key: &TrieH256) {
-		self.overlay.remove(key)
-	}
-}
-
-/// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
-#[derive(Clone)]
-pub struct DbState {
-	db: Arc<KeyValueDB>,
-	root: TrieH256,
-}
-
-impl state_machine::Backend for DbState {
-	type Error = client::error::Error;
-	type Transaction = MemoryDB;
-
-	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		let mut read_overlay = MemoryDB::default();
-		let eph = Ephemeral {
-			backing: &*self.db,
-			overlay: &mut read_overlay,
-		};
-
-		let map_e = |e: Box<TrieError>| ::client::error::Error::from(format!("Trie lookup error: {}", e));
-
-		TrieDB::new(&eph, &self.root).map_err(map_e)?
-			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
-	}
-
-	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
-		let mut read_overlay = MemoryDB::default();
-		let eph = Ephemeral {
-			backing: &*self.db,
-			overlay: &mut read_overlay,
-		};
-
-		let collect_all = || -> Result<_, Box<TrieError>> {
-			let trie = TrieDB::new(&eph, &self.root)?;
-			let mut v = Vec::new();
-			for x in trie.iter()? {
-				let (key, value) = x?;
-				v.push((key.to_vec(), value.to_vec()));
-			}
-
-			Ok(v)
-		};
-
-		match collect_all() {
-			Ok(v) => v,
-			Err(e) => {
-				debug!("Error extracting trie values: {}", e);
-				Vec::new()
-			}
-		}
-	}
-
-	fn storage_root<I>(&self, delta: I) -> ([u8; 32], MemoryDB)
-		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
-	{
-		let mut write_overlay = MemoryDB::default();
-		let mut root = self.root;
-		{
-			let mut eph = Ephemeral {
-				backing: &*self.db,
-				overlay: &mut write_overlay,
-			};
-
-			let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
-			for (key, change) in delta {
-				let result = match change {
-					Some(val) => trie.insert(&key, &val),
-					None => trie.remove(&key), // TODO: archive mode
-				};
-
-				if let Err(e) = result {
-					warn!("Failed to write to trie: {}", e);
-				}
-			}
-		}
-
-		(root.0.into(), write_overlay)
-	}
-}
-
 /// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
 /// Otherwise, trie nodes are kept only from the most recent block.
 pub struct Backend<Block: BlockT> {
@@ -523,25 +394,14 @@ impl<Block: BlockT> client::backend::Backend<Block> for Backend<Block> where
 
 		// special case for genesis initialization
 		match block {
-			BlockId::Hash(h) if h == Default::default() => {
-				let mut root = TrieH256::default();
-				let mut db = MemoryDB::default();
-				TrieDBMut::new(&mut db, &mut root);
-
-				return Ok(DbState {
-					db: self.db.clone(),
-					root,
-				})
-			}
+			BlockId::Hash(h) if h == Default::default() =>
+				return Ok(DbState::with_kvdb_for_genesis(self.db.clone(), ::columns::STATE)),
 			_ => {}
 		}
 
 		self.blockchain.header(block).and_then(|maybe_hdr| maybe_hdr.map(|hdr| {
 			let root: [u8; 32] = hdr.state_root().clone().into();
-			DbState {
-				db: self.db.clone(),
-				root: root.into(),
-			}
+			DbState::with_kvdb(self.db.clone(), ::columns::STATE, root.into())
 		}).ok_or_else(|| client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()))
 	}
 }
@@ -553,6 +413,7 @@ impl<Block: BlockT> client::backend::LocalBackend<Block> for Backend<Block> wher
 
 #[cfg(test)]
 mod tests {
+	use hashdb::HashDB;
 	use super::*;
 	use client::backend::Backend as BTrait;
 	use client::backend::BlockImportOperation as Op;

substrate/client/db/src/lib.rs

@@ -17,9 +17,8 @@
 use std::sync::Arc;
 use futures::{IntoFuture, Future};
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::Block as BlockT;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
 use state_machine::{self, OverlayedChanges, Backend as StateBackend, CodeExecutor};
-use state_machine::backend::InMemory as InMemoryStateBackend;
 
 use backend;
 use blockchain::Backend as ChainBackend;
@@ -49,6 +48,11 @@ pub trait CallExecutor<B: BlockT> {
 	///
 	/// No changes are made.
 	fn call_at_state<S: state_machine::Backend>(&self, state: &S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, S::Transaction), error::Error>;
+
+	/// Execute a call to a contract on top of given state, gathering execution proof.
+	///
+	/// No changes are made.
+	fn prove_at_state<S: state_machine::Backend>(&self, state: S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error>;
 }
 
 /// Call executor that executes methods locally, querying all required
@@ -105,6 +109,18 @@ impl<B, E, Block> CallExecutor<Block> for LocalCallExecutor<B, E>
 			call_data,
 		).map_err(Into::into)
 	}
+
+	fn prove_at_state<S: state_machine::Backend>(&self, state: S, changes: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
+		state_machine::prove(
+			state,
+			changes,
+			&self.executor,
+			method,
+			call_data,
+		)
+		.map(|(result, proof, _)| (result, proof))
+		.map_err(Into::into)
+	}
 }
 
 impl<B, F> RemoteCallExecutor<B, F> {
@@ -140,69 +156,70 @@ impl<B, F, Block> CallExecutor<Block> for RemoteCallExecutor<B, F>
 	fn call_at_state<S: state_machine::Backend>(&self, _state: &S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> error::Result<(Vec<u8>, S::Transaction)> {
 		Err(error::ErrorKind::NotAvailableOnLightClient.into())
 	}
+
+	fn prove_at_state<S: state_machine::Backend>(&self, _state: S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
+		Err(error::ErrorKind::NotAvailableOnLightClient.into())
+	}
 }
 
-/// Check remote execution proof.
-pub fn check_execution_proof<B, E, Block>(backend: &B, executor: &E, request: &RemoteCallRequest<Block::Hash>, remote_proof: (Vec<u8>, Vec<Vec<u8>>)) -> Result<CallResult, error::Error>
+/// Check remote execution proof using given backend.
+pub fn check_execution_proof<B, E, Block>(backend: &B, executor: &E, request: &RemoteCallRequest<Block::Hash>, remote_proof: Vec<Vec<u8>>) -> Result<CallResult, error::Error>
 	where
 		B: backend::RemoteBackend<Block>,
 		E: CodeExecutor,
 		Block: BlockT,
+		<<Block as BlockT>::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic.
 		error::Error: From<<<B as backend::Backend<Block>>::State as StateBackend>::Error>,
 {
-	use runtime_primitives::traits::{Header, Hashing, HashingFor};
-
-	let (remote_result, remote_proof) = remote_proof;
-
-	let remote_state = state_from_execution_proof(remote_proof);
-	let remote_state_root = HashingFor::<Block>::trie_root(remote_state.pairs().into_iter());
-
 	let local_header = backend.blockchain().header(BlockId::Hash(request.block))?;
-	let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", request.block)))?;
+	let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", request.block)))?;
 	let local_state_root = local_header.state_root().clone();
+	do_check_execution_proof(local_state_root, executor, request, remote_proof)
+}
 
-	if remote_state_root != local_state_root {
-		return Err(error::ErrorKind::InvalidExecutionProof.into());
-	}
-
+/// Check remote execution proof using given state root.
+fn do_check_execution_proof<E, H>(local_state_root: H, executor: &E, request: &RemoteCallRequest<H>, remote_proof: Vec<Vec<u8>>) -> Result<CallResult, error::Error>
+	where
+		E: CodeExecutor,
+		H: Into<[u8; 32]>, // TODO: remove when patricia_trie generic.
+{
 	let mut changes = OverlayedChanges::default();
-	let (local_result, _) = state_machine::execute(
-		&remote_state,
+	let (local_result, _) = state_machine::proof_check(
+		local_state_root.into(),
+		remote_proof,
 		&mut changes,
 		executor,
 		&request.method,
-		&request.call_data,
-	)?;
-
-	if local_result != remote_result {
-		return Err(error::ErrorKind::InvalidExecutionProof.into());
-	}
+		&request.call_data)?;
 
 	Ok(CallResult { return_data: local_result, changes })
 }
 
-/// Convert state to execution proof. Proof is simple the whole state (temporary).
-// TODO [light]: this method must be removed after trie-based proofs are landed.
-pub fn state_to_execution_proof<B: state_machine::Backend>(state: &B) -> Vec<Vec<u8>> {
-	state.pairs().into_iter()
-		.flat_map(|(k, v)| ::std::iter::once(k).chain(::std::iter::once(v)))
-		.collect()
-}
-
-/// Convert execution proof to in-memory state for check. Reverse function for state_to_execution_proof.
-// TODO [light]: this method must be removed after trie-based proofs are landed.
-fn state_from_execution_proof(proof: Vec<Vec<u8>>) -> InMemoryStateBackend {
-	let mut changes = Vec::new();
-	let mut proof_iter = proof.into_iter();
-	loop {
-		let key = proof_iter.next();
-		let value = proof_iter.next();
-		if let (Some(key), Some(value)) = (key, value) {
-			changes.push((key, Some(value)));
-		} else {
-			break;
-		}
+#[cfg(test)]
+mod tests {
+	use runtime_primitives::generic::BlockId;
+	use state_machine::Backend;
+	use test_client;
+	use light::RemoteCallRequest;
+	use super::do_check_execution_proof;
+
+	#[test]
+	fn execution_proof_is_generated_and_checked() {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_storage_root = remote_client.state_at(&remote_block_id)
+			.unwrap().storage_root(::std::iter::empty()).0;
+
+		// 'fetch' execution proof from remote node
+		let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1;
+
+		// check remote execution proof locally
+		let local_executor = test_client::NativeExecutor::new();
+		do_check_execution_proof(remote_block_storage_root, &local_executor, &RemoteCallRequest {
+			block: Default::default(),
+			method: "authorities".into(),
+			call_data: vec![],
+		}, remote_execution_proof).unwrap();
 	}
-
-	InMemoryStateBackend::default().update(changes)
 }

substrate/client/src/call_executor.rs

@@ -225,12 +225,7 @@ impl<B, E, Block: BlockT> Client<B, E, Block> where
 	///
 	/// No changes are made.
 	pub fn execution_proof(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> error::Result<(Vec<u8>, Vec<Vec<u8>>)> {
-		use call_executor::state_to_execution_proof;
-
-		let result = self.executor.call(id, method, call_data);
-		let result = result?.return_data;
-		let proof = self.backend.state_at(*id).map(|state| state_to_execution_proof(&state))?;
-		Ok((result, proof))
+		self.state_at(id).and_then(|state| self.executor.prove_at_state(state, &mut Default::default(), method, call_data))
 	}
 
 	/// Set up the native execution environment to call into a native runtime code.