From 7213b1dfcd67679048b365a929fb890d28ded1e3 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 24 Feb 2020 18:21:54 -0300 Subject: [PATCH 1/2] Change from usize to u64 On [random-access-storage](https://github.com/datrs/random-access-storage) issue https://github.com/datrs/random-access-storage/issues/6 has changed all types from `usize` into `u64` to be able to handle more than 4gbs on 32bits systems. > usize is 32 bits on a 32 bit system, so the storage would be limited to 4GB on such a system. When changing `random-access-storage` on `hypercore` (tracking: https://github.com/datrs/hypercore/pull/100) one of the things I've noticed is that it wold be easier to also make `tree-index` use `u64` to integrate with `random-access-storage`. Very likely, this also means that we would need to bump to use more than 32Gb storages on `tree-index`. I've simply changed all declarations of usize into u64. All tests are passing. It also fixes Travis CI and update libraries to latest version. Needs: --- - [ ] https://github.com/datrs/flat-tree/pull/44 --- .travis.yml | 6 ++--- Cargo.toml | 4 ++-- src/lib.rs | 55 +++++++++++++++++++++------------------------ src/proof.rs | 14 ++++++------ src/verification.rs | 4 ++-- tests/test.rs | 20 +++++++++++------ 6 files changed, 52 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b9891b..82f18df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,8 +3,8 @@ cache: cargo rust: stable before_script: - - rustup component add rustfmt-preview - - rustup component add clippy-preview + - rustup component add rustfmt + - rustup component add clippy - cargo fmt --version - cargo clippy --version @@ -12,4 +12,4 @@ script: - cargo fmt -- --check - cargo build --verbose - cargo test --verbose - - cargo clippy + - cargo clippy -- -D clippy::all diff --git a/Cargo.toml b/Cargo.toml index ca97070..0a62315 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,5 +9,5 @@ authors = ["Yoshua Wuyts "] readme = "README.md" [dependencies] -flat-tree = "3.4.0" -sparse-bitfield = "0.8.1" +flat-tree = { version = "4.1.0", git = "https://github.com/bltavares/flat-tree", branch = "usize-to-u64" } +sparse-bitfield = "0.11.0" diff --git a/src/lib.rs b/src/lib.rs index ea78ed9..8de9763 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,21 +44,21 @@ impl TreeIndex { /// Get a bit from the bitfield. #[inline] - pub fn get(&mut self, index: usize) -> bool { - self.bitfield.get(index) + pub fn get(&mut self, index: u64) -> bool { + self.bitfield.get(index as usize) } /// Set an index on the tree to `true`, and also all of the parents to the /// index. Walks the flat-tree upward, until it finds the upper most node. #[inline] - pub fn set(&mut self, mut index: usize) -> Change { - if self.bitfield.set(index, true).is_unchanged() { + pub fn set(&mut self, mut index: u64) -> Change { + if self.bitfield.set(index as usize, true).is_unchanged() { return Change::Unchanged; } - while self.bitfield.get(flat::sibling(index)) { + while self.bitfield.get(flat::sibling(index) as usize) { index = flat::parent(index); - if self.bitfield.set(index, true).is_unchanged() { + if self.bitfield.set(index as usize, true).is_unchanged() { break; } } @@ -69,9 +69,9 @@ impl TreeIndex { #[inline] pub fn proof<'a>( &'a mut self, - index: usize, + index: u64, include_hash: bool, - nodes: &'a mut impl convert::AsMut>, + nodes: &'a mut impl convert::AsMut>, remote_tree: &mut Self, ) -> Option { let digest = 0; @@ -88,10 +88,10 @@ impl TreeIndex { // - opts.digest: not sure what digest does. pub fn proof_with_digest<'a>( &'a mut self, - index: usize, - mut digest: usize, + index: u64, + mut digest: u64, include_hash: bool, - nodes: &'a mut impl convert::AsMut>, + nodes: &'a mut impl convert::AsMut>, remote_tree: &mut Self, ) -> Option { let nodes = nodes.as_mut(); @@ -175,19 +175,17 @@ impl TreeIndex { /// Create a digest for data at index. #[inline] - pub fn digest(&mut self, index: usize) -> usize { + pub fn digest(&mut self, index: u64) -> u64 { if self.get(index) { return 1; } let mut digest = 0; let mut next = flat::sibling(index); - let max = cmp::max(next + 2, self.bitfield.len()); // TODO(from mafintosh/hypercore): make this less hacky + let max = cmp::max(next + 2, self.bitfield.len() as u64); // TODO(from mafintosh/hypercore): make this less hacky let mut bit = 2; - let mut depth = flat::depth(index); - let mut parent = flat::parent_with_depth(next, depth); - depth += 1; + let mut parent = flat::parent(next); while (flat::right_span(next) < max) || flat::left_span(parent) > 0 { if self.get(next) { @@ -203,8 +201,7 @@ impl TreeIndex { } next = flat::sibling(parent); - parent = flat::parent_with_depth(next, depth); - depth += 1; + parent = flat::parent(next); bit *= 2; } digest @@ -238,10 +235,10 @@ impl TreeIndex { /// assert_eq!(tree.blocks(), 4); /// ``` #[inline] - pub fn blocks(&mut self) -> usize { + pub fn blocks(&mut self) -> u64 { let mut top = 0; let mut next = 0; - let max = self.bitfield.len(); + let max = self.bitfield.len() as u64; while flat::right_span(next) < max { next = flat::parent(next); @@ -261,7 +258,7 @@ impl TreeIndex { /// /// TODO: don't make this allocate, but fill a vector instead. #[inline] - pub fn roots(&mut self, roots: &mut Vec) { + pub fn roots(&mut self, roots: &mut Vec) { flat::full_roots(2 * self.blocks(), roots) } @@ -271,7 +268,7 @@ impl TreeIndex { /// push the `top` value into an array, but returns it instead through the /// `Verification` type. #[inline] - pub fn verified_by(&mut self, index: usize) -> Verification { + pub fn verified_by(&mut self, index: u64) -> Verification { let has_index = self.get(index); if !has_index { return Verification { node: 0, top: 0 }; @@ -280,11 +277,11 @@ impl TreeIndex { // Find root of current tree. let mut depth = flat::depth(index); let mut top = index; - let mut parent = flat::parent_with_depth(top, depth); + let mut parent = flat::parent(top); depth += 1; while self.get(parent) && self.get(flat::sibling(top)) { top = parent; - parent = flat::parent_with_depth(top, depth); + parent = flat::parent(top); depth += 1; } @@ -293,14 +290,12 @@ impl TreeIndex { // NOTE: this is probably a candidate to move to `flat-tree`. depth -= 1; while depth != 0 { - top = flat::left_child_with_depth( - flat::index(depth, flat::offset_with_depth(top, depth) + 1), - depth, - ).unwrap(); + top = + flat::left_child(flat::index(depth, flat::offset(top) + 1)).unwrap(); depth -= 1; while !self.get(top) && depth > 0 { - top = flat::left_child_with_depth(top, depth).unwrap(); + top = flat::left_child(top).unwrap(); depth -= 1; } } @@ -324,7 +319,7 @@ impl Default for TreeIndex { /// Check if a value is even. #[inline] -fn is_even(n: usize) -> bool { +fn is_even(n: u64) -> bool { match n & 1 { 0 => true, 1 => false, diff --git a/src/proof.rs b/src/proof.rs index 0e89a7c..8574d64 100644 --- a/src/proof.rs +++ b/src/proof.rs @@ -3,9 +3,9 @@ /// Merkle trees are proven by checking the parent hashes. #[derive(Debug, PartialEq)] pub struct Proof<'a> { - index: usize, - verified_by: usize, - nodes: &'a [usize], + index: u64, + verified_by: u64, + nodes: &'a [u64], } impl<'a> Proof<'a> { @@ -19,7 +19,7 @@ impl<'a> Proof<'a> { /// let _proof = Proof::new(0, 0, &nodes); /// ``` #[inline] - pub fn new(index: usize, verified_by: usize, nodes: &'a [usize]) -> Self { + pub fn new(index: u64, verified_by: u64, nodes: &'a [u64]) -> Self { Self { index, nodes, @@ -38,7 +38,7 @@ impl<'a> Proof<'a> { /// assert_eq!(proof.index(), 0); /// ``` #[inline] - pub fn index(&self) -> usize { + pub fn index(&self) -> u64 { self.index } @@ -53,7 +53,7 @@ impl<'a> Proof<'a> { /// assert_eq!(proof.verified_by(), 0); /// ``` #[inline] - pub fn verified_by(&self) -> usize { + pub fn verified_by(&self) -> u64 { self.verified_by } @@ -68,7 +68,7 @@ impl<'a> Proof<'a> { /// assert_eq!(proof.nodes().len(), 0); /// ``` #[inline] - pub fn nodes(&self) -> &[usize] { + pub fn nodes(&self) -> &[u64] { &self.nodes } } diff --git a/src/verification.rs b/src/verification.rs index b7405a1..b08aa55 100644 --- a/src/verification.rs +++ b/src/verification.rs @@ -2,7 +2,7 @@ #[derive(Debug, PartialEq)] pub struct Verification { /// Node that verifies the index. - pub node: usize, + pub node: u64, /// The highest Node found. - pub top: usize, + pub top: u64, } diff --git a/tests/test.rs b/tests/test.rs index a34f96c..76efd5f 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -96,7 +96,7 @@ fn verified_by() { verify(&mut index, 18, 30, 28); verify(&mut index, 17, 30, 28); - fn verify(tree: &mut TreeIndex, index: usize, node: usize, top: usize) { + fn verify(tree: &mut TreeIndex, index: u64, node: u64, top: u64) { assert_eq!(tree.verified_by(index), Verification { node, top }); } } @@ -282,7 +282,8 @@ fn proof_with_a_digest_2() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![8, 13, 3, 17].as_slice()); assert_eq!(proof.verified_by(), 20); } @@ -295,7 +296,8 @@ fn proof_with_a_digest_2() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![8, 13, 3].as_slice()); assert_eq!(proof.verified_by(), 0); } @@ -308,7 +310,8 @@ fn proof_with_a_digest_2() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![8, 13].as_slice()); assert_eq!(proof.verified_by(), 0); } @@ -321,7 +324,8 @@ fn proof_with_a_digest_2() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![8, 13, 17].as_slice()); assert_eq!(proof.verified_by(), 20); } @@ -353,7 +357,8 @@ fn proof_with_a_digest_3() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![16, 7, 25, 28].as_slice()); assert_eq!(proof.verified_by(), 30); } @@ -374,7 +379,8 @@ fn proof_with_a_digest_3() { false, &mut nodes, &mut TreeIndex::default(), - ).unwrap(); + ) + .unwrap(); assert_eq!(proof.nodes(), vec![21].as_slice()); assert_eq!(proof.verified_by(), 0); } From 0134d747e46d9755d46d2fd009ed1dcd641c9ec1 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 2 Mar 2020 22:16:54 -0300 Subject: [PATCH 2/2] Point flat-tree to crates version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0a62315..bcba9ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,5 +9,5 @@ authors = ["Yoshua Wuyts "] readme = "README.md" [dependencies] -flat-tree = { version = "4.1.0", git = "https://github.com/bltavares/flat-tree", branch = "usize-to-u64" } +flat-tree = "5.0.0" sparse-bitfield = "0.11.0"