From 0bdbf6207af26ca3e3516956db7fa3140679e56e Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Tue, 4 Feb 2020 11:51:33 -0300 Subject: [PATCH 1/6] Bump dalek and rand new dalek pre version has some changes, and requires a new rand version. This commit bumps both, including changes needed on quickcheck to use new rand. --- tests/model.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/model.rs b/tests/model.rs index b7e62fc..5e71999 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -1,12 +1,7 @@ -#[macro_use] -extern crate quickcheck; -extern crate hypercore; -extern crate rand; - mod common; use common::create_feed; -use quickcheck::{Arbitrary, Gen}; +use quickcheck::{quickcheck, Arbitrary, Gen}; use rand::seq::SliceRandom; use rand::Rng; use std::u8; From bd333ba68dc50f6e8bc581d39169ae64f6cba9de Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 7 Feb 2020 18:03:37 -0300 Subject: [PATCH 2/6] Compress bitfield and expose it to network code The network code requires to send compressed bitfields to the wire with the size of the feed. This commit exposes the bitfield as a public method, and adds the compress method. bitfield_rle is not encoding the same as node's version. Tests are broken because of that. --- Cargo.toml | 1 + src/bitfield/mod.rs | 29 +++++++++++++++++++++++++++++ src/feed.rs | 5 +++++ tests/bitfield.rs | 19 +++++++++++++++++++ 4 files changed, 54 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 716635a..7785aa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ sha2 = "0.8.1" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" tree-index = "0.6.0" +bitfield-rle = "0.1.1" [dev-dependencies] quickcheck = "0.9.2" diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index 833cc47..8a65850 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -229,6 +229,35 @@ impl Bitfield { } } + // TODO: use the index to speed this up *a lot* + /// https://github.com/mafintosh/hypercore/blob/06f3a1f573cb74ee8cfab2742455318fbf7cc3a2/lib/bitfield.js#L111-L126 + pub fn compress(&self, start: usize, length: usize) -> std::io::Result> { + if start == 0 && length == 0 { + return Ok(bitfield_rle::encode(&self.data.to_bytes()?)); + } + + use std::io::{Cursor, Write}; + let mut buf = Cursor::new(Vec::with_capacity(length)); + + let page_size = self.data.page_size(); + let p = start / page_size / 8; + let end = p + length / page_size / 8; + let offset = p * page_size; + + for _ in p..end { + let page = self.data.pages.get(p); + if let Some(page) = page { + if page.len() == 0 { + continue; + } + buf.set_position((p * page_size - offset) as u64); + buf.write_all(&page)?; + } + } + + Ok(bitfield_rle::encode(&buf.into_inner())) + } + /// Constructs an iterator from start to end pub fn iterator(&mut self) -> iterator::Iterator<'_> { let len = self.length; diff --git a/src/feed.rs b/src/feed.rs index a1433ea..34a4232 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -524,6 +524,11 @@ where }) } + /// Expose the bitfield attribute to use on during download + pub fn bitfield(&self) -> &Bitfield { + &self.bitfield + } + /// (unimplemented) Provide a range of data to download. pub fn download(&mut self, _range: Range) -> Result<()> { unimplemented!(); diff --git a/tests/bitfield.rs b/tests/bitfield.rs index bc02642..90a436c 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -174,3 +174,22 @@ fn bitfield_dedup() { assert!(!b.get(8 * 1024)); assert!(b.get(16 * 1024)); } + +#[test] +fn bitfield_compress() { + let mut b = Bitfield::new(); + assert_eq!(b.compress(0, 0).unwrap(), vec![]); + + b.set(1, true); + assert_eq!(b.compress(0, 0).unwrap(), vec![2, 64]); + + for i in 0..32 { + b.set(i, true); + } + assert_eq!(b.compress(0, 0).unwrap(), vec![19]); + assert_eq!(b.compress(0, 1).unwrap(), vec![7]); + assert_eq!(b.compress(0, 2).unwrap(), vec![11]); + assert_eq!(b.compress(0, 3).unwrap(), vec![15]); + assert_eq!(b.compress(0, 4).unwrap(), vec![19]); + assert_eq!(b.compress(0, 5).unwrap(), vec![19]); +} From 390e13f9b527845f281b24071bbf579f9a6232eb Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 7 Feb 2020 20:39:56 -0300 Subject: [PATCH 3/6] WIP: JS has float numbers on math --- src/bitfield/mod.rs | 19 ++++++++++--------- tests/bitfield.rs | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index 8a65850..b6b3b2b 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -239,20 +239,21 @@ impl Bitfield { use std::io::{Cursor, Write}; let mut buf = Cursor::new(Vec::with_capacity(length)); - let page_size = self.data.page_size(); - let p = start / page_size / 8; - let end = p + length / page_size / 8; + let page_size = self.data.page_size() as f64; + let mut p = start as f64 / page_size / 8_f64; + let end = p + length as f64 / page_size / 8_f64; let offset = p * page_size; - for _ in p..end { - let page = self.data.pages.get(p); + while p < end { + let index = p as usize; + let page = self.data.pages.get(index); if let Some(page) = page { - if page.len() == 0 { - continue; + if dbg!(page.len()) != 0 { + buf.set_position((p * page_size - offset) as u64); + buf.write_all(&page)?; } - buf.set_position((p * page_size - offset) as u64); - buf.write_all(&page)?; } + p += 1_f64; } Ok(bitfield_rle::encode(&buf.into_inner())) diff --git a/tests/bitfield.rs b/tests/bitfield.rs index 90a436c..61306d3 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -178,18 +178,41 @@ fn bitfield_dedup() { #[test] fn bitfield_compress() { let mut b = Bitfield::new(); - assert_eq!(b.compress(0, 0).unwrap(), vec![]); + assert_eq!(b.compress(0, 0).unwrap(), vec![0]); b.set(1, true); - assert_eq!(b.compress(0, 0).unwrap(), vec![2, 64]); + assert_eq!(b.compress(0, 0).unwrap(), vec![2, 64, 253, 31]); for i in 0..32 { b.set(i, true); } - assert_eq!(b.compress(0, 0).unwrap(), vec![19]); - assert_eq!(b.compress(0, 1).unwrap(), vec![7]); - assert_eq!(b.compress(0, 2).unwrap(), vec![11]); - assert_eq!(b.compress(0, 3).unwrap(), vec![15]); - assert_eq!(b.compress(0, 4).unwrap(), vec![19]); - assert_eq!(b.compress(0, 5).unwrap(), vec![19]); + + for i in 64..1024 { + b.set(i, true); + } + + for i in 1024..1028 { + b.set(i, true); + } + + assert_eq!( + b.compress(0, 0).unwrap(), + vec![19, 17, 227, 3, 2, 240, 253, 27] + ); + assert_eq!( + b.compress(0, 1).unwrap(), + vec![19, 17, 227, 3, 2, 240, 253, 27] + ); + assert_eq!( + b.compress(0, 32).unwrap(), + vec![19, 17, 227, 3, 2, 240, 253, 27] + ); + assert_eq!( + b.compress(0, 1024).unwrap(), + vec![19, 17, 227, 3, 2, 240, 253, 27] + ); + assert_eq!( + b.compress(1024, 2048).unwrap(), + vec![19, 17, 227, 3, 2, 240, 253, 27] + ); } From 356c90e915a9a5dcc4edb5bf0fa61eda200f6b9b Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 7 Feb 2020 20:58:46 -0300 Subject: [PATCH 4/6] Make test with bigger ranges than page size --- src/bitfield/mod.rs | 2 +- tests/bitfield.rs | 34 +++++++--------------------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index b6b3b2b..0111dd6 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -248,7 +248,7 @@ impl Bitfield { let index = p as usize; let page = self.data.pages.get(index); if let Some(page) = page { - if dbg!(page.len()) != 0 { + if page.len() != 0 { buf.set_position((p * page_size - offset) as u64); buf.write_all(&page)?; } diff --git a/tests/bitfield.rs b/tests/bitfield.rs index 61306d3..b90ee7d 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -183,36 +183,16 @@ fn bitfield_compress() { b.set(1, true); assert_eq!(b.compress(0, 0).unwrap(), vec![2, 64, 253, 31]); - for i in 0..32 { - b.set(i, true); - } - - for i in 64..1024 { - b.set(i, true); - } - - for i in 1024..1028 { - b.set(i, true); - } - + b.set(1_424_244, true); assert_eq!( b.compress(0, 0).unwrap(), - vec![19, 17, 227, 3, 2, 240, 253, 27] - ); - assert_eq!( - b.compress(0, 1).unwrap(), - vec![19, 17, 227, 3, 2, 240, 253, 27] - ); - assert_eq!( - b.compress(0, 32).unwrap(), - vec![19, 17, 227, 3, 2, 240, 253, 27] - ); - assert_eq!( - b.compress(0, 1024).unwrap(), - vec![19, 17, 227, 3, 2, 240, 253, 27] + vec![2, 64, 181, 187, 43, 2, 8, 197, 4] ); + assert_eq!(b.compress(0, 1).unwrap(), vec![2, 64, 253, 31]); assert_eq!( - b.compress(1024, 2048).unwrap(), - vec![19, 17, 227, 3, 2, 240, 253, 27] + b.compress(1_424_244, 1).unwrap(), + vec![185, 27, 2, 8, 197, 4] ); + + assert_eq!(b.compress(1_424_244_000, 1).unwrap(), vec![0]); } From 9c6812d901454a383bee9802e0f5828c3224b515 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Fri, 7 Feb 2020 20:59:54 -0300 Subject: [PATCH 5/6] Use literals for floats --- src/bitfield/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index 0111dd6..6f6399f 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -240,8 +240,8 @@ impl Bitfield { let mut buf = Cursor::new(Vec::with_capacity(length)); let page_size = self.data.page_size() as f64; - let mut p = start as f64 / page_size / 8_f64; - let end = p + length as f64 / page_size / 8_f64; + let mut p = start as f64 / page_size / 8.0; + let end = p + length as f64 / page_size / 8.0; let offset = p * page_size; while p < end { @@ -253,7 +253,7 @@ impl Bitfield { buf.write_all(&page)?; } } - p += 1_f64; + p += 1.0; } Ok(bitfield_rle::encode(&buf.into_inner())) From d8beadbbfb0ff7d2d79e52abc14ffb570570b101 Mon Sep 17 00:00:00 2001 From: Bruno Tavares Date: Mon, 2 Mar 2020 22:35:25 -0300 Subject: [PATCH 6/6] GH Feedback: add comments on the optional fields --- src/bitfield/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index 6f6399f..367c319 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -232,6 +232,8 @@ impl Bitfield { // TODO: use the index to speed this up *a lot* /// https://github.com/mafintosh/hypercore/blob/06f3a1f573cb74ee8cfab2742455318fbf7cc3a2/lib/bitfield.js#L111-L126 pub fn compress(&self, start: usize, length: usize) -> std::io::Result> { + // On Node versions this fields might not be present on the want/request message + // When both start and length are not present (!0 in node is false), return all data bytes encoded if start == 0 && length == 0 { return Ok(bitfield_rle::encode(&self.data.to_bytes()?)); }