Update snapshot message format

Author: remagpie

spec/Block-Synchronization-Extension.md

@@ -53,32 +53,17 @@ Request corresponding bodies for each hash.
 * Restriction:
   * MUST include at least one item
 
-
-### GetStateHead
-
-```
-GetStateHead(block_hash)
-```
-
-Request corresponding state head for block of `block_hash`.
-
-* Identifier: 0x06
-* Restriction: Block number of requested block MUST be multiple of 214.
-
-
 ### GetStateChunk
 
 ```
-GetStateChunk(block_hash, tree_root)
+GetStateChunk(block_hash, [...chunk_roots])
 ```
 
-Request entire subtree starting from `tree_root`.
+Request corresponding snapshot chunk for each `chunk_root`.
 
-* Identifier: 0x08
+* Identifier: 0x0a
 * Restriction:
-  * Block number of requested block MUST be multiple of 214.
-  * `tree_root` MUST be included in requested block’s state trie.
-  * Depth of `tree_root` inside state trie MUST be equal to 2. (Depth of state root is 0)
+  * All values in `[...chunk_roots]` MUST be included in requested block’s state trie.
 
 
 ## Response messages
@@ -113,30 +98,15 @@ Response to `GetBodies` message. Snappy algorithm is used to compress content.
   * If received body is zero-length array, it means either body value is [], or sender doesn’t have body for requested hash
 
 
-### StateHead
-
-```
-StateHead(compressed((key_0, value_0), …) | [])
-```
-
-Response to `GetStateHead` message. Key and value included in this messages are raw value stored in state trie. Snappy algorithm is used for compression of content.
-
-* Identifier: 0x07
-* Restriction:
-  * State root of requested block MUST be included
-  * For all nodes with depth of less than 2 included in this message, all of its child MUST also be included.
-  * Content MUST be empty array if sender didn’t have requested data
-
-
 ### StateChunk
 ```
-StateChunk(compressed((key_0, value_0), …) | [])
+StateChunk([compressed([terminal_0, …] | []), ...])
 ```
 
-Response to `GetStateChunk` message. Details of message is same as `StateHead` message.
+Response to `GetStateChunk` message. Snappy algorithm is used for compression of content.
 
-* Identifier: 0x09
+* Identifier: 0x0b
 * Restriction:
-  * Node corresponding to tree_root in request MUST be included
-  * Every nodes included in message MUST have all of its child in same message.
-  * Content MUST be empty array if sender didn’t have requested data
+  * Number and order of chunks included in this message MUST be equal to request information.
+  * Node corresponding to `chunk_root` in request MUST be included
+  * If sender doesn’t have a chunk for the requested hash, corresponding chunk MUST be compressed([]), not omitted.

sync/src/block/extension.rs

@@ -49,8 +49,6 @@ const SYNC_EXPIRE_TOKEN_END: TimerToken = SYNC_EXPIRE_TOKEN_BEGIN + SYNC_EXPIRE_
 const SYNC_TIMER_INTERVAL: u64 = 1000;
 const SYNC_EXPIRE_REQUEST_INTERVAL: u64 = 15000;
 
-const SNAPSHOT_PERIOD: u64 = (1 << 14);
-
 #[derive(Debug, PartialEq)]
 pub struct TokenInfo {
     node_id: NodeId,
@@ -563,11 +561,9 @@ impl Extension {
                 ctrace!(SYNC, "Received body request from {}", from);
                 self.create_bodies_response(hashes)
             }
-            RequestMessage::StateHead(hash) => self.create_state_head_response(hash),
-            RequestMessage::StateChunk {
-                block_hash,
-                tree_root,
-            } => self.create_state_chunk_response(block_hash, tree_root),
+            RequestMessage::StateChunk(block_hash, chunk_root) => {
+                self.create_state_chunk_response(block_hash, chunk_root)
+            }
         };
 
         self.api.send(from, Arc::new(Message::Response(id, response).rlp_bytes()));
@@ -579,21 +575,9 @@ impl Extension {
                 ..
             } => true,
             RequestMessage::Bodies(hashes) => !hashes.is_empty(),
-            RequestMessage::StateHead(hash) => match self.client.block_number(&BlockId::Hash(*hash)) {
-                Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
-                _ => false,
-            },
             RequestMessage::StateChunk {
-                block_hash,
                 ..
-            } => {
-                let _is_checkpoint = match self.client.block_number(&BlockId::Hash(*block_hash)) {
-                    Some(number) if number % SNAPSHOT_PERIOD == 0 => true,
-                    _ => false,
-                };
-                // FIXME:  check tree_root
-                unimplemented!()
-            }
+            } => unimplemented!(),
         }
     }
 
@@ -631,11 +615,7 @@ impl Extension {
         ResponseMessage::Bodies(bodies)
     }
 
-    fn create_state_head_response(&self, _hash: BlockHash) -> ResponseMessage {
-        unimplemented!()
-    }
-
-    fn create_state_chunk_response(&self, _hash: BlockHash, _tree_root: H256) -> ResponseMessage {
+    fn create_state_chunk_response(&self, _hash: BlockHash, _tree_root: Vec<H256>) -> ResponseMessage {
         unimplemented!()
     }
 
@@ -676,7 +656,7 @@ impl Extension {
                     self.on_body_response(hashes, bodies);
                     self.check_sync_variable();
                 }
-                _ => unimplemented!(),
+                ResponseMessage::StateChunk(..) => unimplemented!(),
             }
         }
     }
@@ -730,7 +710,6 @@ impl Extension {
                 }
                 true
             }
-            (RequestMessage::StateHead(..), ResponseMessage::StateHead(..)) => unimplemented!(),
             (
                 RequestMessage::StateChunk {
                     ..

sync/src/block/message/mod.rs

@@ -29,10 +29,8 @@ const MESSAGE_ID_GET_HEADERS: u8 = 0x02;
 const MESSAGE_ID_HEADERS: u8 = 0x03;
 const MESSAGE_ID_GET_BODIES: u8 = 0x04;
 const MESSAGE_ID_BODIES: u8 = 0x05;
-const MESSAGE_ID_GET_STATE_HEAD: u8 = 0x06;
-const MESSAGE_ID_STATE_HEAD: u8 = 0x07;
-const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x08;
-const MESSAGE_ID_STATE_CHUNK: u8 = 0x09;
+const MESSAGE_ID_GET_STATE_CHUNK: u8 = 0x0a;
+const MESSAGE_ID_STATE_CHUNK: u8 = 0x0b;
 
 #[derive(Debug, PartialEq)]
 pub enum Message {
@@ -114,11 +112,10 @@ impl Decodable for Message {
             let request_id = rlp.val_at(1)?;
             let message = rlp.at(2)?;
             match id {
-                MESSAGE_ID_GET_HEADERS
-                | MESSAGE_ID_GET_BODIES
-                | MESSAGE_ID_GET_STATE_HEAD
-                | MESSAGE_ID_GET_STATE_CHUNK => Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?)),
-                MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_HEAD | MESSAGE_ID_STATE_CHUNK => {
+                MESSAGE_ID_GET_HEADERS | MESSAGE_ID_GET_BODIES | MESSAGE_ID_GET_STATE_CHUNK => {
+                    Ok(Message::Request(request_id, RequestMessage::decode(id, &message)?))
+                }
+                MESSAGE_ID_HEADERS | MESSAGE_ID_BODIES | MESSAGE_ID_STATE_CHUNK => {
                     Ok(Message::Response(request_id, ResponseMessage::decode(id, &message)?))
                 }
                 _ => Err(DecoderError::Custom("Unknown message id detected")),
@@ -148,10 +145,4 @@ mod tests {
         let request_id = 10;
         rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::Bodies(vec![])));
     }
-
-    #[test]
-    fn request_state_head_rlp() {
-        let request_id = 10;
-        rlp_encode_and_decode_test!(Message::Request(request_id, RequestMessage::StateHead(H256::random().into())));
-    }
 }

sync/src/block/message/request.rs

@@ -25,11 +25,7 @@ pub enum RequestMessage {
         max_count: u64,
     },
     Bodies(Vec<BlockHash>),
-    StateHead(BlockHash),
-    StateChunk {
-        block_hash: BlockHash,
-        tree_root: H256,
-    },
+    StateChunk(BlockHash, Vec<H256>),
 }
 
 impl Encodable for RequestMessage {
@@ -46,17 +42,10 @@ impl Encodable for RequestMessage {
             RequestMessage::Bodies(hashes) => {
                 s.append_list(hashes);
             }
-            RequestMessage::StateHead(block_hash) => {
-                s.begin_list(1);
-                s.append(block_hash);
-            }
-            RequestMessage::StateChunk {
-                block_hash,
-                tree_root,
-            } => {
+            RequestMessage::StateChunk(block_hash, merkle_roots) => {
                 s.begin_list(2);
                 s.append(block_hash);
-                s.append(tree_root);
+                s.append_list(merkle_roots);
             }
         };
     }
@@ -69,7 +58,6 @@ impl RequestMessage {
                 ..
             } => super::MESSAGE_ID_GET_HEADERS,
             RequestMessage::Bodies(..) => super::MESSAGE_ID_GET_BODIES,
-            RequestMessage::StateHead(..) => super::MESSAGE_ID_GET_STATE_HEAD,
             RequestMessage::StateChunk {
                 ..
             } => super::MESSAGE_ID_GET_STATE_CHUNK,
@@ -92,16 +80,6 @@ impl RequestMessage {
                 }
             }
             super::MESSAGE_ID_GET_BODIES => RequestMessage::Bodies(rlp.as_list()?),
-            super::MESSAGE_ID_GET_STATE_HEAD => {
-                let item_count = rlp.item_count()?;
-                if item_count != 1 {
-                    return Err(DecoderError::RlpIncorrectListLen {
-                        got: item_count,
-                        expected: 1,
-                    })
-                }
-                RequestMessage::StateHead(rlp.val_at(0)?)
-            }
             super::MESSAGE_ID_GET_STATE_CHUNK => {
                 let item_count = rlp.item_count()?;
                 if item_count != 2 {
@@ -110,10 +88,7 @@ impl RequestMessage {
                         expected: 2,
                     })
                 }
-                RequestMessage::StateChunk {
-                    block_hash: rlp.val_at(0)?,
-                    tree_root: rlp.val_at(1)?,
-                }
+                RequestMessage::StateChunk(rlp.val_at(0)?, rlp.list_at(1)?)
             }
             _ => return Err(DecoderError::Custom("Unknown message id detected")),
         };
@@ -149,18 +124,9 @@ mod tests {
         assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
     }
 
-    #[test]
-    fn request_state_head_message_rlp() {
-        let message = RequestMessage::StateHead(H256::default().into());
-        assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
-    }
-
     #[test]
     fn request_state_chunk_message_rlp() {
-        let message = RequestMessage::StateChunk {
-            block_hash: H256::default().into(),
-            tree_root: H256::default(),
-        };
+        let message = RequestMessage::StateChunk(H256::default().into(), vec![H256::default()]);
         assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
     }
 }

sync/src/block/message/response.rs

@@ -24,8 +24,7 @@ use ctypes::Header;
 pub enum ResponseMessage {
     Headers(Vec<Header>),
     Bodies(Vec<Vec<UnverifiedTransaction>>),
-    StateHead(Vec<u8>),
-    StateChunk(Vec<u8>),
+    StateChunk(Vec<Vec<u8>>),
 }
 
 impl Encodable for ResponseMessage {
@@ -53,13 +52,8 @@ impl Encodable for ResponseMessage {
 
                 s.append(&compressed);
             }
-            ResponseMessage::StateHead(bytes) => {
-                s.begin_list(1);
-                s.append(bytes);
-            }
-            ResponseMessage::StateChunk(bytes) => {
-                s.begin_list(1);
-                s.append(bytes);
+            ResponseMessage::StateChunk(chunks) => {
+                s.append_list::<Vec<u8>, Vec<u8>>(chunks);
             }
         };
     }
@@ -72,7 +66,6 @@ impl ResponseMessage {
                 ..
             } => super::MESSAGE_ID_HEADERS,
             ResponseMessage::Bodies(..) => super::MESSAGE_ID_BODIES,
-            ResponseMessage::StateHead(..) => super::MESSAGE_ID_STATE_HEAD,
             ResponseMessage::StateChunk {
                 ..
             } => super::MESSAGE_ID_STATE_CHUNK,
@@ -109,26 +102,7 @@ impl ResponseMessage {
                 }
                 ResponseMessage::Bodies(bodies)
             }
-            super::MESSAGE_ID_STATE_HEAD => {
-                let item_count = rlp.item_count()?;
-                if item_count != 1 {
-                    return Err(DecoderError::RlpIncorrectListLen {
-                        got: item_count,
-                        expected: 1,
-                    })
-                }
-                ResponseMessage::StateHead(rlp.val_at(0)?)
-            }
-            super::MESSAGE_ID_STATE_CHUNK => {
-                let item_count = rlp.item_count()?;
-                if item_count != 1 {
-                    return Err(DecoderError::RlpIncorrectListLen {
-                        got: item_count,
-                        expected: 1,
-                    })
-                }
-                ResponseMessage::StateChunk(rlp.val_at(0)?)
-            }
+            super::MESSAGE_ID_STATE_CHUNK => ResponseMessage::StateChunk(rlp.as_list()?),
             _ => return Err(DecoderError::Custom("Unknown message id detected")),
         };
 
@@ -184,12 +158,6 @@ mod tests {
         assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
     }
 
-    #[test]
-    fn state_head_message_rlp() {
-        let message = ResponseMessage::StateHead(vec![]);
-        assert_eq!(message, decode_bytes(message.message_id(), message.rlp_bytes().as_ref()));
-    }
-
     #[test]
     fn state_chunk_message_rlp() {
         let message = ResponseMessage::StateChunk(vec![]);

util/merkle/src/snapshot/mod.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-mod chunk;
+pub mod chunk;
 mod compress;
 mod error;
 mod ordered_heap;