From ac3790dd4da0c72341944f29a75a8bf1fefcae00 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 21 Feb 2020 23:36:36 -0300 Subject: [PATCH 1/8] Bump versions to latest versions --- Cargo.toml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 581cf63..ec218cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,25 +19,25 @@ edition = "2018" [dependencies] blake2-rfc = "0.2.18" -byteorder = "1.3.2" -ed25519-dalek = "0.9.1" -failure = "0.1.5" +byteorder = "1.3.4" +ed25519-dalek = "1.0.0-pre.3" +failure = "0.1.6" flat-tree = "4.1.0" -lazy_static = "1.3.0" +lazy_static = "1.4.0" memory-pager = "0.9.0" merkle-tree-stream = "0.11.0" -pretty-hash = "0.4.0" -rand = "0.6.0" -random-access-disk = "0.8.0" -random-access-memory = "1.0.0" -random-access-storage = "2.0.0" -sha2 = "0.8.0" +pretty-hash = "0.4.1" +rand = "0.7.3" +random-access-disk = "1.0.0" +random-access-memory = "1.1.0" +random-access-storage = "3.0.0" +sha2 = "0.8.1" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" tree-index = "0.5.0" [dev-dependencies] -quickcheck = "0.8.5" +quickcheck = "0.9.2" data-encoding = "2.1.2" remove_dir_all = "0.5.2" tempfile = "3.1.0" From c4dc33a69aeead974d7dbd35d8414016ea3e421b Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 21 Feb 2020 23:36:36 -0300 Subject: [PATCH 2/8] Bump versions to latest versions --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ec218cd..6e809c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ tree-index = "0.5.0" [dev-dependencies] quickcheck = "0.9.2" -data-encoding = "2.1.2" +data-encoding = "2.2.0" remove_dir_all = "0.5.2" tempfile = "3.1.0" async-std = "1.5.0" From 7fd467d92800e00cff7600fe6e68fbb474c899be Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Sun, 23 Feb 2020 10:42:11 -0300 Subject: [PATCH 3/8] Fix Travis config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b9891b..82f18df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ cache: cargo rust: stable before_script: - - rustup component add rustfmt-preview - - rustup component add clippy-preview + - rustup component add rustfmt + - rustup component add clippy - cargo fmt --version - cargo clippy --version @@ -12,4 +12,4 @@ script: - cargo fmt -- --check - cargo build --verbose - cargo test --verbose - - cargo clippy + - cargo clippy -- -D clippy::all From 0678d066875b7cef8cde3628f7ef91658a40f8c1 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Sun, 23 Feb 2020 10:44:59 -0300 Subject: [PATCH 4/8] Fix changes on ed25519_dalek and rand --- src/crypto/key_pair.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index 48d6337..6a41ed5 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -1,19 +1,20 @@ //! Generate an `Ed25519` keypair. -pub use ed25519_dalek::{Keypair, PublicKey, SecretKey, Signature}; +pub use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature}; use crate::Result; -use rand::rngs::OsRng; -use sha2::Sha512; +use rand::rngs::{OsRng, StdRng}; +use rand::SeedableRng; /// Generate a new `Ed25519` key pair. pub fn generate() -> Keypair { - Keypair::generate::(&mut OsRng::new().unwrap()) + let mut rng = StdRng::from_rng(OsRng::default()).unwrap(); + Keypair::generate(&mut rng) } /// Sign a byte slice using a keypair's private key. pub fn sign(public_key: &PublicKey, secret: &SecretKey, msg: &[u8]) -> Signature { - secret.expand::().sign::(msg, public_key) + ExpandedSecretKey::from(secret).sign(msg, public_key) } /// Verify a signature on a message with a keypair's public key. @@ -22,7 +23,7 @@ pub fn verify(public: &PublicKey, msg: &[u8], sig: Option<&Signature>) -> Result None => bail!("Signature verification failed"), Some(sig) => { ensure!( - public.verify::(msg, sig).is_ok(), + public.verify(msg, sig).is_ok(), "Signature verification failed" ); Ok(()) From 173bc3fda2f079994a38577030142b97c3143b4f Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 24 Feb 2020 10:07:51 -0300 Subject: [PATCH 5/8] Move from usize to u64 This commit compiles `hypercore` using the new u64 crates: - random-access-storage - random-access-disk - random-access-memmory - merkle-tree-stream - flat-tree - index-tree --- Cargo.toml | 6 +-- examples/iter.rs | 6 +-- src/audit.rs | 8 ++-- src/bitfield/iterator.rs | 37 ++++++++------- src/bitfield/mod.rs | 100 ++++++++++++++++++++------------------- src/crypto/hash.rs | 8 ++-- src/crypto/root.rs | 62 ++++++++++++------------ src/feed.rs | 48 +++++++++---------- src/proof.rs | 4 +- src/replicate/message.rs | 4 +- src/replicate/peer.rs | 4 +- src/storage/mod.rs | 42 ++++++++-------- src/storage/node.rs | 16 +++---- src/storage/persist.rs | 4 +- tests/bitfield.rs | 8 ++-- tests/model.rs | 12 ++--- 16 files changed, 187 insertions(+), 182 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6e809c1..b182e4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,10 +22,10 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.0-pre.3" failure = "0.1.6" -flat-tree = "4.1.0" +flat-tree = { version = "4.1.0", git = "https://github.com/bltavares/flat-tree", branch = "usize-to-u64" } lazy_static = "1.4.0" memory-pager = "0.9.0" -merkle-tree-stream = "0.11.0" +merkle-tree-stream = { version = "0.11.0", git = "https://github.com/bltavares/merkle-tree-stream", branch = "usize-to-u64" } pretty-hash = "0.4.1" rand = "0.7.3" random-access-disk = "1.0.0" @@ -34,7 +34,7 @@ random-access-storage = "3.0.0" sha2 = "0.8.1" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" -tree-index = "0.5.0" +tree-index = { version = "0.5.0", git = "https://github.com/bltavares/tree-index", branch = "usize-to-u64" } [dev-dependencies] quickcheck = "0.9.2" diff --git a/examples/iter.rs b/examples/iter.rs index 21beadd..d3353c5 100644 --- a/examples/iter.rs +++ b/examples/iter.rs @@ -13,7 +13,7 @@ struct BookShelf { #[derive(Debug)] struct BookShelfIterator<'b> { /// Keeps track which index we're currently at. - pub cursor: usize, + pub cursor: u64, /// Borrow of the Bookshelf we're going to iterate over. pub inner: &'b BookShelf, } @@ -35,10 +35,10 @@ impl<'b> iter::Iterator for BookShelfIterator<'b> { let cursor = self.cursor; self.cursor += 1; - if cursor >= self.inner.books.len() { + if cursor >= self.inner.books.len() as u64 { None } else { - Some(&self.inner.books[cursor]) + Some(&self.inner.books[cursor as usize]) } } } diff --git a/src/audit.rs b/src/audit.rs index c56aa97..ef6f9a4 100644 --- a/src/audit.rs +++ b/src/audit.rs @@ -2,19 +2,19 @@ #[derive(Debug, PartialEq, Clone)] pub struct Audit { /// The number of valid blocks identified - pub valid_blocks: usize, + pub valid_blocks: u64, /// The number of invalid blocks identified - pub invalid_blocks: usize, + pub invalid_blocks: u64, } impl Audit { /// Access the `valid_blocks` field from the proof. - pub fn valid_blocks(&self) -> usize { + pub fn valid_blocks(&self) -> u64 { self.valid_blocks } /// Access the `invalid_blocks` field from the proof. - pub fn invalid_blocks(&self) -> usize { + pub fn invalid_blocks(&self) -> u64 { self.invalid_blocks } } diff --git a/src/bitfield/iterator.rs b/src/bitfield/iterator.rs index a4ef6e9..2c6d3d9 100644 --- a/src/bitfield/iterator.rs +++ b/src/bitfield/iterator.rs @@ -5,10 +5,10 @@ use super::Bitfield; /// Iterate over a bitfield. #[derive(Debug)] pub struct Iterator<'a> { - start: usize, - end: usize, - index_end: usize, - pos: Option, + start: u64, + end: u64, + index_end: u64, + pos: Option, byte: u8, bitfield: &'a mut Bitfield, } @@ -27,7 +27,7 @@ impl<'a> Iterator<'a> { } /// Grow the bitfield if needed. - pub fn range(&mut self, start: usize, end: usize) { + pub fn range(&mut self, start: u64, end: u64) { self.start = start; self.end = end; self.index_end = 2 * ((end + 31) / 32); @@ -38,7 +38,7 @@ impl<'a> Iterator<'a> { } /// Seek to `offset` - pub fn seek(&mut self, mut offset: usize) -> &mut Self { + pub fn seek(&mut self, mut offset: u64) -> &mut Self { offset += self.start; // FIXME This is fishy. Offset and start is unsigned, so `offset < self.start` can only // be true when the previous addition overflows. The overflow would cause a panic, so, @@ -58,25 +58,26 @@ impl<'a> Iterator<'a> { let pos = offset / 8; self.pos = Some(pos); - self.byte = self.bitfield.data.get_byte(pos) | self.bitfield.masks.data_iterate[o]; + self.byte = self.bitfield.data.get_byte(pos as usize) + | self.bitfield.masks.data_iterate[o as usize]; self } - pub fn next(&mut self) -> Option { + pub fn next(&mut self) -> Option { let mut pos = self.pos?; let mut free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; while free == -1 { pos += 1; - self.byte = self.bitfield.data.get_byte(pos); + self.byte = self.bitfield.data.get_byte(pos as usize); free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; if free == -1 { pos = self.skip_ahead(pos)?; - self.byte = self.bitfield.data.get_byte(pos); + self.byte = self.bitfield.data.get_byte(pos as usize); free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; } } @@ -84,7 +85,7 @@ impl<'a> Iterator<'a> { self.byte |= self.bitfield.masks.data_iterate[free as usize + 1]; - let n = 8 * pos + free as usize; + let n = 8 * pos + free as u64; if n < self.end { Some(n) } else { @@ -92,7 +93,7 @@ impl<'a> Iterator<'a> { } } - pub fn skip_ahead(&mut self, start: usize) -> Option { + pub fn skip_ahead(&mut self, start: u64) -> Option { let bitfield_index = &self.bitfield.index; let tree_end = self.index_end; let iter = &mut self.bitfield.iterator; @@ -100,8 +101,8 @@ impl<'a> Iterator<'a> { iter.seek(2 * (start / 4)); - let mut tree_byte = - bitfield_index.get_byte(iter.index()) | self.bitfield.masks.index_iterate[o]; + let mut tree_byte = bitfield_index.get_byte(iter.index() as usize) + | self.bitfield.masks.index_iterate[o as usize]; while self.bitfield.masks.next_index_0_bit[tree_byte as usize] == -1 { if iter.is_left() { @@ -120,7 +121,7 @@ impl<'a> Iterator<'a> { } } - tree_byte = bitfield_index.get_byte(iter.index()); + tree_byte = bitfield_index.get_byte(iter.index() as usize); } while iter.factor() > 2 { @@ -130,7 +131,7 @@ impl<'a> Iterator<'a> { iter.right_child(); } - tree_byte = bitfield_index.get_byte(iter.index()); + tree_byte = bitfield_index.get_byte(iter.index() as usize); } let mut free = self.bitfield.masks.next_index_0_bit[tree_byte as usize]; @@ -138,7 +139,7 @@ impl<'a> Iterator<'a> { free = 4; } - let next = iter.index() * 2 + free as usize; + let next = iter.index() * 2 + free as u64; if next <= start { Some(start + 1) @@ -148,7 +149,7 @@ impl<'a> Iterator<'a> { } } -fn right_span(iter: &flat_tree::Iterator) -> usize { +fn right_span(iter: &flat_tree::Iterator) -> u64 { iter.index() + iter.factor() / 2 - 1 } diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index 2811956..833cc47 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -32,8 +32,8 @@ pub struct Bitfield { /// FIXME: SLEEP protocol tree field. pub tree: SparseBitfield, index: SparseBitfield, - page_len: usize, - length: usize, + page_len: u64, + length: u64, masks: Masks, iterator: FlatIterator, } @@ -59,7 +59,7 @@ impl Bitfield { } /// Get the current length - pub fn len(&self) -> usize { + pub fn len(&self) -> u64 { self.length } @@ -69,44 +69,44 @@ impl Bitfield { } /// Set a value at an index. - pub fn set(&mut self, index: usize, value: bool) -> Change { + pub fn set(&mut self, index: u64, value: bool) -> Change { let o = mask_8b(index); let index = (index - o) / 8; let value = if value { - self.data.get_byte(index) | 128 >> o + self.data.get_byte(index as usize) | 128 >> o } else { - self.data.get_byte(index) & self.masks.data_update[o] + self.data.get_byte(index as usize) & self.masks.data_update[o as usize] }; - if self.data.set_byte(index, value).is_unchanged() { + if self.data.set_byte(index as usize, value).is_unchanged() { return Change::Unchanged; } - self.length = self.data.len(); + self.length = self.data.len() as u64; self.set_index(index, value); Change::Changed } /// Get a value at a position in the bitfield. - pub fn get(&mut self, index: usize) -> bool { - self.data.get(index) + pub fn get(&mut self, index: u64) -> bool { + self.data.get(index as usize) } /// Calculate the total for the whole data. pub fn total(&mut self) -> u8 { - let len = self.data.len(); + let len = self.data.len() as u64; self.total_with_range(0..len) } /// Calculate the total of ... TODO(yw) - pub fn total_with_start(&mut self, start: usize) -> u8 { - let len = self.data.len(); + pub fn total_with_start(&mut self, start: u64) -> u8 { + let len = self.data.len() as u64; self.total_with_range(start..len) } /// Calculate the total of ... TODO(yw) - pub fn total_with_range(&mut self, range: Range) -> u8 { + pub fn total_with_range(&mut self, range: Range) -> u8 { let start = range.start; let end = range.end; @@ -114,7 +114,7 @@ impl Bitfield { return 0; } - if end > self.data.len() { + if end > self.data.len() as u64 { self.expand(end); } @@ -124,24 +124,24 @@ impl Bitfield { let pos = (start - o) / 8; let last = (end - e) / 8; - let left_mask = 255 - self.masks.data_iterate[o]; - let right_mask = self.masks.data_iterate[e]; + let left_mask = 255 - self.masks.data_iterate[o as usize]; + let right_mask = self.masks.data_iterate[e as usize]; - let byte = self.data.get_byte(pos); + let byte = self.data.get_byte(pos as usize); if pos == last { - let index = (byte & left_mask & right_mask) as usize; - return self.masks.total_1_bits[index]; + let index = (byte & left_mask & right_mask) as u64; + return self.masks.total_1_bits[index as usize]; } - let index = (byte & left_mask) as usize; - let mut total = self.masks.total_1_bits[index]; + let index = (byte & left_mask) as u64; + let mut total = self.masks.total_1_bits[index as usize]; for i in pos + 1..last { - let index = self.data.get_byte(i) as usize; - total += self.masks.total_1_bits[index]; + let index = self.data.get_byte(i as usize) as u64; + total += self.masks.total_1_bits[index as usize]; } - let index: usize = self.data.get_byte(last) as usize & right_mask as usize; - total + self.masks.total_1_bits[index] + let index: u64 = self.data.get_byte(last as usize) as u64 & right_mask as u64; + total + self.masks.total_1_bits[index as usize] } /// Set a value at index. @@ -153,13 +153,13 @@ impl Bitfield { /// /// NOTE(yw): lots of magic values going on; I have no idea what we're doing /// here. - fn set_index(&mut self, mut index: usize, value: u8) -> Change { + fn set_index(&mut self, mut index: u64, value: u8) -> Change { let o = index & 3; index = (index - o) / 4; let start = tree_index(index); - let left = self.index.get_byte(start) & self.masks.index_update[o]; + let left = self.index.get_byte(start as usize) & self.masks.index_update[o as usize]; let right = get_index_value(value) >> tree_index(o); let mut byte = left | right; let len = self.index.len(); @@ -170,26 +170,26 @@ impl Bitfield { while self.iterator.index() < max_len && self .index - .set_byte(self.iterator.index(), byte) + .set_byte(self.iterator.index() as usize, byte) .is_changed() { if self.iterator.is_left() { - let index: usize = self.index.get_byte(self.iterator.sibling()).into(); - byte = - self.masks.map_parent_left[byte as usize] | self.masks.map_parent_right[index]; + let index: u64 = self.index.get_byte(self.iterator.sibling() as usize).into(); + byte = self.masks.map_parent_left[byte as usize] + | self.masks.map_parent_right[index as usize]; } else { - let index: usize = self + let index: u64 = self .index - .get_byte(self.iterator.sibling()) // FIXME: out of bounds read + .get_byte(self.iterator.sibling() as usize) // FIXME: out of bounds read .into(); - byte = - self.masks.map_parent_right[byte as usize] | self.masks.map_parent_left[index]; + byte = self.masks.map_parent_right[byte as usize] + | self.masks.map_parent_left[index as usize]; } self.iterator.parent(); } if len != self.index.len() { - self.expand(len); + self.expand(len as u64); } if self.iterator.index() == start { @@ -199,7 +199,7 @@ impl Bitfield { } } - fn expand(&mut self, len: usize) { + fn expand(&mut self, len: u64) { let mut roots = vec![]; // FIXME: alloc. flat_tree::full_roots(tree_index(len), &mut roots); let bf = &mut self.index; @@ -209,15 +209,17 @@ impl Bitfield { for root in roots { ite.seek(root); - byte = bf.get_byte(ite.index()); + byte = bf.get_byte(ite.index() as usize); loop { if ite.is_left() { - let index = bf.get_byte(ite.sibling()) as usize; - byte = masks.map_parent_left[byte as usize] | masks.map_parent_right[index]; + let index = bf.get_byte(ite.sibling() as usize) as u64; + byte = masks.map_parent_left[byte as usize] + | masks.map_parent_right[index as usize]; } else { - let index = bf.get_byte(ite.sibling()) as usize; - byte = masks.map_parent_right[byte as usize] | masks.map_parent_left[index]; + let index = bf.get_byte(ite.sibling() as usize) as u64; + byte = masks.map_parent_right[byte as usize] + | masks.map_parent_left[index as usize]; } if set_byte_no_alloc(bf, ite.parent(), byte).is_unchanged() { @@ -234,7 +236,7 @@ impl Bitfield { } /// Constructs an iterator from `start` to `end` - pub fn iterator_with_range(&mut self, start: usize, end: usize) -> iterator::Iterator<'_> { + pub fn iterator_with_range(&mut self, start: u64, end: u64) -> iterator::Iterator<'_> { let mut iter = iterator::Iterator::new(self); iter.range(start, end); iter.seek(0); @@ -244,11 +246,11 @@ impl Bitfield { } // NOTE: can we move this into `sparse_bitfield`? -fn set_byte_no_alloc(bf: &mut SparseBitfield, index: usize, byte: u8) -> Change { - if 8 * index >= bf.len() { +fn set_byte_no_alloc(bf: &mut SparseBitfield, index: u64, byte: u8) -> Change { + if 8 * index >= bf.len() as u64 { return Change::Unchanged; } - bf.set_byte(index, byte) + bf.set_byte(index as usize, byte) } #[inline] @@ -261,12 +263,12 @@ fn get_index_value(index: u8) -> u8 { } #[inline] -fn mask_8b(num: usize) -> usize { +fn mask_8b(num: u64) -> u64 { num & 7 } /// Convert the index to the index in the tree. #[inline] -fn tree_index(index: usize) -> usize { +fn tree_index(index: u64) -> u64 { 2 * index } diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 69d1a5c..95b552d 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -142,8 +142,8 @@ mod tests { fn parent_hash() { let d1: &[u8] = &[0, 1, 2, 3, 4]; let d2: &[u8] = &[42, 43, 44, 45, 46, 47, 48]; - let node1 = Node::new(0, Hash::from_leaf(d1).as_bytes().to_vec(), d1.len()); - let node2 = Node::new(1, Hash::from_leaf(d2).as_bytes().to_vec(), d2.len()); + let node1 = Node::new(0, Hash::from_leaf(d1).as_bytes().to_vec(), d1.len() as u64); + let node2 = Node::new(1, Hash::from_leaf(d2).as_bytes().to_vec(), d2.len() as u64); check_hash( Hash::from_hashes(&node1, &node2), "6fac58578fa385f25a54c0637adaca71fdfddcea885d561f33d80c4487149a14", @@ -158,8 +158,8 @@ mod tests { fn root_hash() { let d1: &[u8] = &[0, 1, 2, 3, 4]; let d2: &[u8] = &[42, 43, 44, 45, 46, 47, 48]; - let node1 = Node::new(0, Hash::from_leaf(d1).as_bytes().to_vec(), d1.len()); - let node2 = Node::new(1, Hash::from_leaf(d2).as_bytes().to_vec(), d2.len()); + let node1 = Node::new(0, Hash::from_leaf(d1).as_bytes().to_vec(), d1.len() as u64); + let node2 = Node::new(1, Hash::from_leaf(d2).as_bytes().to_vec(), d2.len() as u64); check_hash( Hash::from_roots(&[&node1, &node2]), "2d117e0bb15c6e5236b6ce764649baed1c41890da901a015341503146cc20bcd", diff --git a/src/crypto/root.rs b/src/crypto/root.rs index 5087db0..12a6713 100644 --- a/src/crypto/root.rs +++ b/src/crypto/root.rs @@ -10,43 +10,43 @@ /// Root node found in flat-tree. pub struct Root<'a> { - index: &'a usize, - length: &'a usize, - hash: &'a [u8], + index: &'a u64, + length: &'a u64, + hash: &'a [u8], } impl<'a> Root<'a> { - /// Create a new instance. - #[inline] - pub fn new(index: &'a usize, length: &'a usize, hash: &'a [u8]) -> Self { - Self { - index, - length, - hash, + /// Create a new instance. + #[inline] + pub fn new(index: &'a u64, length: &'a u64, hash: &'a [u8]) -> Self { + Self { + index, + length, + hash, + } } - } - /// Get the index at which this root was found inside a `flat-tree`. - #[inline] - pub fn index(&self) -> &usize { - &self.index - } + /// Get the index at which this root was found inside a `flat-tree`. + #[inline] + pub fn index(&self) -> &u64 { + &self.index + } - /// Get the lenght of the data. - #[inline] - pub fn len(&self) -> &usize { - &self.length - } + /// Get the lenght of the data. + #[inline] + pub fn len(&self) -> &u64 { + &self.length + } - /// Check if the content is empty. - #[inline] - pub fn is_empty(&self) -> bool { - *self.length == 0 - } + /// Check if the content is empty. + #[inline] + pub fn is_empty(&self) -> bool { + *self.length == 0 + } - /// Get the hash. - #[inline] - pub fn hash(&self) -> &'a [u8] { - &self.hash - } + /// Get the hash. + #[inline] + pub fn hash(&self) -> &'a [u8] { + &self.hash + } } diff --git a/src/feed.rs b/src/feed.rs index 225ff98..a1433ea 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -37,9 +37,9 @@ where pub(crate) secret_key: Option, pub(crate) storage: Storage, /// Total length of data stored. - pub(crate) byte_length: usize, + pub(crate) byte_length: u64, /// TODO: description. Length of... roots? - pub(crate) length: usize, + pub(crate) length: u64, /// Bitfield to keep track of which data we own. pub(crate) bitfield: Bitfield, pub(crate) tree: TreeIndex, @@ -85,7 +85,7 @@ where /// Get the amount of entries in the feed. #[inline] - pub fn len(&self) -> usize { + pub fn len(&self) -> u64 { self.length } @@ -97,7 +97,7 @@ where /// Get the total amount of bytes stored in the feed. #[inline] - pub fn byte_len(&self) -> usize { + pub fn byte_len(&self) -> u64 { self.byte_length } @@ -112,7 +112,7 @@ where let mut offset = 0; self.storage.write_data(self.byte_length + offset, &data)?; - offset += data.len(); + offset += data.len() as u64; let hash = Hash::from_roots(self.merkle.roots()); let index = self.length; @@ -144,26 +144,26 @@ where /// Return `true` if a data block is available locally. #[inline] - pub fn has(&mut self, index: usize) -> bool { + pub fn has(&mut self, index: u64) -> bool { self.bitfield.get(index) } /// Return `true` if all data blocks within a range are available locally. #[inline] - pub fn has_all(&mut self, range: ::std::ops::Range) -> bool { - let total = range.len(); + pub fn has_all(&mut self, range: ::std::ops::Range) -> bool { + let total = range.clone().count(); total == self.bitfield.total_with_range(range) as usize } /// Get the total amount of chunks downloaded. #[inline] - pub fn downloaded(&mut self, range: ::std::ops::Range) -> u8 { + pub fn downloaded(&mut self, range: ::std::ops::Range) -> u8 { self.bitfield.total_with_range(range) } /// Retrieve data from the log. #[inline] - pub fn get(&mut self, index: usize) -> Result>> { + pub fn get(&mut self, index: u64) -> Result>> { if !self.bitfield.get(index) { // NOTE: Do (network) lookup here once we have network code. return Ok(None); @@ -173,7 +173,7 @@ where /// Return the Nodes which prove the correctness for the Node at index. #[inline] - pub fn proof(&mut self, index: usize, include_hash: bool) -> Result { + pub fn proof(&mut self, index: u64, include_hash: bool) -> Result { self.proof_with_digest(index, 0, include_hash) } @@ -181,8 +181,8 @@ where /// digest. pub fn proof_with_digest( &mut self, - index: usize, - digest: usize, + index: u64, + digest: u64, include_hash: bool, ) -> Result { let mut remote_tree = TreeIndex::default(); @@ -226,16 +226,16 @@ where } /// Compute the digest for the index. - pub fn digest(&mut self, index: usize) -> usize { + pub fn digest(&mut self, index: u64) -> u64 { self.tree.digest(tree_index(index)) } /// Insert data into the tree at `index`. Verifies the `proof` when inserting /// to make sure data is correct. Useful when replicating data from a remote /// host. - pub fn put(&mut self, index: usize, data: Option<&[u8]>, mut proof: Proof) -> Result<()> { + pub fn put(&mut self, index: u64, data: Option<&[u8]>, mut proof: Proof) -> Result<()> { let mut next = tree_index(index); - let mut trusted: Option = None; + let mut trusted: Option = None; let mut missing = vec![]; let mut i = match data { @@ -281,7 +281,7 @@ where Some(data) => Node::new( tree_index(index), Hash::from_leaf(&data).as_bytes().to_owned(), - data.len(), + data.len() as u64, ), None => proof.nodes.remove(0), }; @@ -339,7 +339,7 @@ where // Arguments are: (index, data, node, sig, from, cb) fn write( &mut self, - index: usize, + index: u64, data: Option<&[u8]>, nodes: &[Node], sig: Option, @@ -386,7 +386,7 @@ where } /// Get a signature from the store. - pub fn signature(&mut self, index: usize) -> Result { + pub fn signature(&mut self, index: u64) -> Result { ensure!( index < self.length, format!("No signature found for index {}", index) @@ -396,7 +396,7 @@ where /// Verify the entire feed. Checks a signature against the signature of all /// root nodes combined. - pub fn verify(&mut self, index: usize, signature: &Signature) -> Result<()> { + pub fn verify(&mut self, index: u64, signature: &Signature) -> Result<()> { let roots = self.root_hashes(index)?; let roots: Vec<_> = roots.into_iter().map(Arc::new).collect(); @@ -428,7 +428,7 @@ where /// Get all root hashes from the feed. // In the JavaScript implementation this calls to `._getRootsToVerify()` // internally. In Rust it seems better to just inline the code. - pub fn root_hashes(&mut self, index: usize) -> Result> { + pub fn root_hashes(&mut self, index: u64) -> Result> { ensure!( index <= self.length, format!("Root index bounds exceeded {} > {}", index, self.length) @@ -525,12 +525,12 @@ where } /// (unimplemented) Provide a range of data to download. - pub fn download(&mut self, _range: Range) -> Result<()> { + pub fn download(&mut self, _range: Range) -> Result<()> { unimplemented!(); } /// (unimplemented) Provide a range of data to remove from the local storage. - pub fn undownload(&mut self, _range: Range) -> Result<()> { + pub fn undownload(&mut self, _range: Range) -> Result<()> { unimplemented!(); } @@ -594,6 +594,6 @@ impl + Debug> Display for Feed { /// Convert the index to the index in the tree. #[inline] -fn tree_index(index: usize) -> usize { +fn tree_index(index: u64) -> u64 { 2 * index } diff --git a/src/proof.rs b/src/proof.rs index ef9ec5a..e7d4cd4 100644 --- a/src/proof.rs +++ b/src/proof.rs @@ -5,7 +5,7 @@ use crate::Signature; #[derive(Debug, PartialEq, Clone)] pub struct Proof { /// The index to which this proof corresponds. - pub index: usize, + pub index: u64, /// Nodes that verify the index you passed. pub nodes: Vec, /// An `ed25519` signature, guaranteeing the integrity of the nodes. @@ -14,7 +14,7 @@ pub struct Proof { impl Proof { /// Access the `index` field from the proof. - pub fn index(&self) -> usize { + pub fn index(&self) -> u64 { self.index } diff --git a/src/replicate/message.rs b/src/replicate/message.rs index 19a1226..8a8887e 100644 --- a/src/replicate/message.rs +++ b/src/replicate/message.rs @@ -1,6 +1,6 @@ /// A message sent over the network. #[derive(Debug, Clone, PartialEq)] pub struct Message { - start: usize, - length: Option, + start: u64, + length: Option, } diff --git a/src/replicate/peer.rs b/src/replicate/peer.rs index dc00ffa..58e9357 100644 --- a/src/replicate/peer.rs +++ b/src/replicate/peer.rs @@ -10,8 +10,8 @@ use super::Message; // ] #[derive(Debug, Clone, PartialEq)] pub struct Peer { - // remote_id: usize, -// remote_length: usize, + // remote_id: u64, +// remote_length: u64, // remote_bitfield: Bitfield, // remote_is_want: bool, // remote_is_downloading: bool, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 7db2885..4959611 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -20,7 +20,7 @@ use std::fmt::Debug; use std::ops::Range; use std::path::PathBuf; -const HEADER_OFFSET: usize = 32; +const HEADER_OFFSET: u64 = 32; #[derive(Debug)] pub struct PartialKeypair { @@ -90,7 +90,7 @@ where /// Write data to the feed. #[inline] - pub fn write_data(&mut self, offset: usize, data: &[u8]) -> Result<()> { + pub fn write_data(&mut self, offset: u64, data: &[u8]) -> Result<()> { self.data.write(offset, &data) } @@ -102,16 +102,16 @@ where /// with mafintosh). /// TODO: Ensure the signature size is correct. /// NOTE: Should we create a `Data` entry type? - pub fn put_data(&mut self, index: usize, data: &[u8], nodes: &[Node]) -> Result<()> { + pub fn put_data(&mut self, index: u64, data: &[u8], nodes: &[Node]) -> Result<()> { if data.is_empty() { return Ok(()); } - let range = self.data_offset(index, nodes)?; + let mut range = self.data_offset(index, nodes)?; ensure!( - range.len() == data.len(), - format!("length `{:?} != {:?}`", range.len(), data.len()) + range.by_ref().count() == data.len(), + format!("length `{:?} != {:?}`", range.count(), data.len()) ); self.data.write(range.start, data) @@ -121,14 +121,14 @@ where /// unencrypted, so there's no decryption needed. // FIXME: data_offset always reads out index 0, length 0 #[inline] - pub fn get_data(&mut self, index: usize) -> Result> { + pub fn get_data(&mut self, index: u64) -> Result> { let cached_nodes = Vec::new(); // TODO: reuse allocation. let range = self.data_offset(index, &cached_nodes)?; - self.data.read(range.start, range.len()) + self.data.read(range.start, range.count() as u64) } /// Search the signature stores for a `Signature`, starting at `index`. - pub fn next_signature(&mut self, index: usize) -> Result { + pub fn next_signature(&mut self, index: u64) -> Result { let bytes = self.signatures.read(HEADER_OFFSET + 64 * index, 64)?; if not_zeroes(&bytes) { Ok(Signature::from_bytes(&bytes)?) @@ -139,7 +139,7 @@ where /// Get a `Signature` from the store. #[inline] - pub fn get_signature(&mut self, index: usize) -> Result { + pub fn get_signature(&mut self, index: u64) -> Result { let bytes = self.signatures.read(HEADER_OFFSET + 64 * index, 64)?; ensure!(not_zeroes(&bytes), "No signature found"); Ok(Signature::from_bytes(&bytes)?) @@ -149,7 +149,7 @@ where /// TODO: Ensure the signature size is correct. /// NOTE: Should we create a `Signature` entry type? #[inline] - pub fn put_signature(&mut self, index: usize, signature: impl Borrow) -> Result<()> { + pub fn put_signature(&mut self, index: u64, signature: impl Borrow) -> Result<()> { let signature = signature.borrow(); self.signatures .write(HEADER_OFFSET + 64 * index, &signature.to_bytes()) @@ -160,12 +160,12 @@ where /// /// ## Panics /// A panic can occur if no maximum value is found. - pub fn data_offset(&mut self, index: usize, cached_nodes: &[Node]) -> Result> { + pub fn data_offset(&mut self, index: u64, cached_nodes: &[Node]) -> Result> { let mut roots = Vec::new(); // TODO: reuse alloc flat::full_roots(tree_index(index), &mut roots); let mut offset = 0; - let mut pending = roots.len(); + let mut pending = roots.len() as u64; let block_index = tree_index(index); if pending == 0 { @@ -207,7 +207,7 @@ where /// Get a `Node` from the `tree` storage. #[inline] - pub fn get_node(&mut self, index: usize) -> Result { + pub fn get_node(&mut self, index: u64) -> Result { let buf = self.tree.read(HEADER_OFFSET + 40 * index, 40)?; let node = Node::from_bytes(index, &buf)?; Ok(node) @@ -227,20 +227,22 @@ where /// TODO: Ensure the chunk size is correct. /// NOTE: Should we create a bitfield entry type? #[inline] - pub fn put_bitfield(&mut self, offset: usize, data: &[u8]) -> Result<()> { + pub fn put_bitfield(&mut self, offset: u64, data: &[u8]) -> Result<()> { self.bitfield.write(HEADER_OFFSET + offset, data) } /// Read a public key from storage pub fn read_public_key(&mut self) -> Result { - let buf = self.keypair.read(0, PUBLIC_KEY_LENGTH)?; + let buf = self.keypair.read(0, PUBLIC_KEY_LENGTH as u64)?; let public_key = PublicKey::from_bytes(&buf)?; Ok(public_key) } /// Read a secret key from storage pub fn read_secret_key(&mut self) -> Result { - let buf = self.keypair.read(PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH)?; + let buf = self + .keypair + .read(PUBLIC_KEY_LENGTH as u64, SECRET_KEY_LENGTH as u64)?; let secret_key = SecretKey::from_bytes(&buf)?; Ok(secret_key) } @@ -254,7 +256,7 @@ where /// Write a secret key to the storage pub fn write_secret_key(&mut self, secret_key: &SecretKey) -> Result<()> { let buf: [u8; SECRET_KEY_LENGTH] = secret_key.to_bytes(); - self.keypair.write(PUBLIC_KEY_LENGTH, &buf) + self.keypair.write(PUBLIC_KEY_LENGTH as u64, &buf) } /// Tries to read a partial keypair (ie: with an optional secret_key) from the storage @@ -302,7 +304,7 @@ impl Storage { /// Get a node from a vector of nodes. #[inline] -fn find_node(nodes: &[Node], index: usize) -> Option<&Node> { +fn find_node(nodes: &[Node], index: u64) -> Option<&Node> { for node in nodes { if node.index() == index { return Some(node); @@ -324,7 +326,7 @@ fn not_zeroes(bytes: &[u8]) -> bool { /// Convert the index to the index in the tree. #[inline] -fn tree_index(index: usize) -> usize { +fn tree_index(index: u64) -> u64 { 2 * index } diff --git a/src/storage/node.rs b/src/storage/node.rs index eb948bb..c774688 100644 --- a/src/storage/node.rs +++ b/src/storage/node.rs @@ -16,17 +16,17 @@ use crate::crypto::Hash; // disk. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Node { - pub(crate) index: usize, + pub(crate) index: u64, pub(crate) hash: Vec, pub(crate) length: u64, - pub(crate) parent: usize, + pub(crate) parent: u64, pub(crate) data: Option>, } impl Node { /// Create a new instance. // TODO: ensure sizes are correct. - pub fn new(index: usize, hash: Vec, length: usize) -> Self { + pub fn new(index: u64, hash: Vec, length: u64) -> Self { Self { index, hash, @@ -39,7 +39,7 @@ impl Node { /// Convert a vector to a new instance. /// /// Requires the index at which the buffer was read to be passed. - pub fn from_bytes(index: usize, buffer: &[u8]) -> Result { + pub fn from_bytes(index: u64, buffer: &[u8]) -> Result { ensure!(buffer.len() == 40, "buffer should be 40 bytes"); let parent = flat_tree::parent(index); @@ -73,7 +73,7 @@ impl Node { impl NodeTrait for Node { #[inline] - fn index(&self) -> usize { + fn index(&self) -> u64 { self.index } @@ -83,8 +83,8 @@ impl NodeTrait for Node { } #[inline] - fn len(&self) -> usize { - self.length as usize + fn len(&self) -> u64 { + self.length as u64 } #[inline] @@ -93,7 +93,7 @@ impl NodeTrait for Node { } #[inline] - fn parent(&self) -> usize { + fn parent(&self) -> u64 { self.parent } } diff --git a/src/storage/persist.rs b/src/storage/persist.rs index b576e12..55e5c9a 100644 --- a/src/storage/persist.rs +++ b/src/storage/persist.rs @@ -9,11 +9,11 @@ where T: RandomAccess + Debug, { /// Create an instance from a byte vector. - fn from_bytes(index: usize, buf: &[u8]) -> Self; + fn from_bytes(index: u64, buf: &[u8]) -> Self; /// Create a vector. fn to_vec(&self) -> Result>; /// Persist into a storage backend. - fn store(&self, index: usize, store: Storage) -> Result<()>; + fn store(&self, index: u64, store: Storage) -> Result<()>; } diff --git a/tests/bitfield.rs b/tests/bitfield.rs index c9395a3..bc02642 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -112,18 +112,18 @@ fn set_and_index_random() { assert!(check(&mut b), "index validates"); fn check(b: &mut Bitfield) -> bool { - let mut all = vec![true; b.len()]; + let mut all = vec![true; b.len() as usize]; { let mut iter = b.iterator(); while let Some(i) = iter.next() { - all[i] = false; + all[i as usize] = false; } } for (i, &v) in all.iter().enumerate() { - if b.get(i) != v { + if b.get(i as u64) != v { return false; } } @@ -131,7 +131,7 @@ fn set_and_index_random() { true } - fn set(b: &mut Bitfield, i: usize, n: usize) { + fn set(b: &mut Bitfield, i: u64, n: u64) { for j in i..i + n { b.set(j, true); } diff --git a/tests/model.rs b/tests/model.rs index 1d69ad9..b7e62fc 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -11,11 +11,11 @@ use rand::seq::SliceRandom; use rand::Rng; use std::u8; -const MAX_FILE_SIZE: usize = 5 * 10; // 5mb +const MAX_FILE_SIZE: u64 = 5 * 10; // 5mb #[derive(Clone, Debug)] enum Op { - Get { index: usize }, + Get { index: u64 }, Append { data: Vec }, Verify, } @@ -25,12 +25,12 @@ impl Arbitrary for Op { let choices = [0, 1, 2]; match choices.choose(g).expect("Value should exist") { 0 => { - let index: usize = g.gen_range(0, MAX_FILE_SIZE); + let index: u64 = g.gen_range(0, MAX_FILE_SIZE); Op::Get { index } } 1 => { - let length: usize = g.gen_range(0, MAX_FILE_SIZE / 3); - let mut data = Vec::with_capacity(length); + let length: u64 = g.gen_range(0, MAX_FILE_SIZE / 3); + let mut data = Vec::with_capacity(length as usize); for _ in 0..length { data.push(u8::arbitrary(g)); } @@ -61,7 +61,7 @@ quickcheck! { if index >= insta.len() { assert_eq!(data, None); } else { - assert_eq!(data, Some(model[index].clone())); + assert_eq!(data, Some(model[index as usize].clone())); } }, Op::Verify => { From ba09c2733684f0320a7f99ebfa3ec8aae31334fd Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 24 Feb 2020 18:47:41 -0300 Subject: [PATCH 6/8] Fix travis: include checks on benchmarks --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 82f18df..d14d3af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,11 @@ before_script: - rustup component add clippy - cargo fmt --version - cargo clippy --version + - rustup update nightly script: - cargo fmt -- --check + - cargo +nightly check --all-targets - cargo build --verbose - cargo test --verbose - cargo clippy -- -D clippy::all From f3b421c6ca76a0b5c5acb267988d97ba97e8a77a Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 24 Feb 2020 18:52:44 -0300 Subject: [PATCH 7/8] Fix clippy: rename func to adhere to conventions --- src/crypto/hash.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 95b552d..dc8ba2a 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -59,7 +59,7 @@ impl Hash { /// Hash a public key. Useful to find the key you're looking for on a public /// network without leaking the key itself. - pub fn to_discovery_key(public_key: PublicKey) -> Self { + pub fn for_discovery_key(public_key: PublicKey) -> Self { let mut hasher = Blake2b::with_key(32, public_key.as_bytes()); hasher.update(&HYPERCORE); Self { @@ -182,7 +182,7 @@ mod tests { 59, 1, 248, 146, 32, 159, 121, 183, 90, 87, 217, 137, 225, ]; - assert_eq!(Hash::to_discovery_key(public_key).as_bytes(), expected); + assert_eq!(Hash::for_discovery_key(public_key).as_bytes(), expected); Ok(()) } From 51c35d8f42c42e111f2c207f1901288aaee7e500 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 2 Mar 2020 22:20:38 -0300 Subject: [PATCH 8/8] Point deps to crates versions --- Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b182e4d..a3b9c53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,10 +22,10 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.0-pre.3" failure = "0.1.6" -flat-tree = { version = "4.1.0", git = "https://github.com/bltavares/flat-tree", branch = "usize-to-u64" } +flat-tree = "5.0.0" lazy_static = "1.4.0" memory-pager = "0.9.0" -merkle-tree-stream = { version = "0.11.0", git = "https://github.com/bltavares/merkle-tree-stream", branch = "usize-to-u64" } +merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" random-access-disk = "1.0.0" @@ -34,7 +34,7 @@ random-access-storage = "3.0.0" sha2 = "0.8.1" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" -tree-index = { version = "0.5.0", git = "https://github.com/bltavares/tree-index", branch = "usize-to-u64" } +tree-index = "0.6.0" [dev-dependencies] quickcheck = "0.9.2"