Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ edition = "2018"
blake2-rfc = "0.2.18"
byteorder = "1.3.4"
ed25519-dalek = "1.0.0-pre.3"
failure = "0.1.6"
anyhow = "1.0.26"
flat-tree = "5.0.0"
lazy_static = "1.4.0"
memory-pager = "0.9.0"
merkle-tree-stream = "0.12.0"
pretty-hash = "0.4.1"
rand = "0.7.3"
random-access-disk = "~1.0.0"
random-access-memory = "~1.1.0"
random-access-disk = "1.1.0"
random-access-memory = "1.2.0"
random-access-storage = "3.0.0"
sha2 = "0.8.1"
sleep-parser = "0.8.0"
Expand Down
12 changes: 4 additions & 8 deletions benches/bench.rs
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
#![feature(test)]

extern crate failure;
extern crate hypercore;
extern crate random_access_memory as ram;

extern crate test;

use self::test::Bencher;
use failure::Error;
use anyhow::Error;
use random_access_memory::RandomAccessMemory;
use test::Bencher;

use hypercore::{Feed, Storage, Store};
use ram::RandomAccessMemory;

fn create_feed(page_size: usize) -> Result<Feed<RandomAccessMemory>, Error> {
let create = |_store: Store| Ok(RandomAccessMemory::new(page_size));
Expand Down
59 changes: 29 additions & 30 deletions examples/async.rs
Original file line number Diff line number Diff line change
@@ -1,30 +1,29 @@
use async_std::task;
use failure::Error;
use hypercore::Feed;
use random_access_storage::RandomAccess;
use std::fmt::Debug;

async fn append<T>(feed: &mut Feed<T>, content: &[u8])
where
T: RandomAccess<Error = Error> + Debug,
{
feed.append(content).unwrap();
}

async fn print<T>(feed: &mut Feed<T>)
where
T: RandomAccess<Error = Error> + Debug,
{
println!("{:?}", feed.get(0));
println!("{:?}", feed.get(1));
}

fn main() {
task::block_on(task::spawn(async {
let mut feed = Feed::default();

append(&mut feed, b"hello").await;
append(&mut feed, b"world").await;
print(&mut feed).await;
}));
}
use async_std::task;
use hypercore::Feed;
use random_access_storage::RandomAccess;
use std::fmt::Debug;

async fn append<T>(feed: &mut Feed<T>, content: &[u8])
where
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe define an alias in the crate root?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've wanted to do that, but trait alias are only available on nightly:

error[E0658]: trait aliases are experimental
 --> examples\async.rs:6:1
  |
6 | trait Storage = RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug + Send;
  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  |
  = note: for more information, see https://github.com/rust-lang/rust/issues/41517

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll open an issue to track rust-lang/rust#41517 and move it on after this lands.

{
feed.append(content).unwrap();
}

async fn print<T>(feed: &mut Feed<T>)
where
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
{
println!("{:?}", feed.get(0));
println!("{:?}", feed.get(1));
}

fn main() {
task::block_on(task::spawn(async {
let mut feed = Feed::default();

append(&mut feed, b"hello").await;
append(&mut feed, b"world").await;
print(&mut feed).await;
}));
}
2 changes: 1 addition & 1 deletion examples/iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ struct BookShelfIterator<'b> {

impl BookShelf {
/// Return an iterator over all values.
pub fn iter(&self) -> BookShelfIterator {
pub fn iter(&self) -> BookShelfIterator<'_> {
BookShelfIterator {
inner: self,
cursor: 0,
Expand Down
2 changes: 0 additions & 2 deletions examples/main.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
extern crate hypercore;

use hypercore::Feed;

fn main() {
Expand Down
2 changes: 1 addition & 1 deletion src/crypto/key_pair.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

pub use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature};

use crate::Result;
use anyhow::{bail, ensure, Result};
use rand::rngs::{OsRng, StdRng};
use rand::SeedableRng;

Expand Down
11 changes: 6 additions & 5 deletions src/feed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,8 @@ use crate::audit::Audit;
use crate::bitfield::Bitfield;
use crate::crypto::{generate_keypair, sign, verify, Hash, Merkle};
use crate::proof::Proof;
use crate::Result;
use anyhow::{bail, ensure, Result};
use ed25519_dalek::{PublicKey, SecretKey, Signature};
use failure::Error;
use flat_tree as flat;
use pretty_hash::fmt as pretty_fmt;
use random_access_disk::RandomAccessDisk;
Expand All @@ -29,7 +28,7 @@ use std::sync::Arc;
#[derive(Debug)]
pub struct Feed<T>
where
T: RandomAccess<Error = Error> + Debug,
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
{
/// Merkle tree instance.
pub(crate) merkle: Merkle,
Expand All @@ -48,7 +47,7 @@ where

impl<T> Feed<T>
where
T: RandomAccess<Error = Error> + Debug,
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
{
/// Create a new instance with a custom storage backend.
pub fn with_storage(mut storage: crate::storage::Storage<T>) -> Result<Self> {
Expand Down Expand Up @@ -581,7 +580,9 @@ impl Default for Feed<RandomAccessMemory> {
}
}

impl<T: RandomAccess<Error = Error> + Debug> Display for Feed<T> {
impl<T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug> Display
for Feed<T>
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// TODO: yay, we should find a way to convert this .unwrap() to an error
// type that's accepted by `fmt::Result<(), fmt::Error>`.
Expand Down
5 changes: 2 additions & 3 deletions src/feed_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,12 @@ use ed25519_dalek::{PublicKey, SecretKey};
use crate::bitfield::Bitfield;
use crate::crypto::Merkle;
use crate::storage::Storage;
use failure::Error;
use random_access_storage::RandomAccess;
use std::fmt::Debug;
use tree_index::TreeIndex;

use crate::Feed;
use crate::Result;
use anyhow::Result;

/// Construct a new `Feed` instance.
// TODO: make this an actual builder pattern.
Expand All @@ -26,7 +25,7 @@ where

impl<T> FeedBuilder<T>
where
T: RandomAccess<Error = Error> + Debug,
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
{
/// Create a new instance.
#[inline]
Expand Down
12 changes: 2 additions & 10 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
//!
//! ## Example
//! ```rust
//! # fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync + 'static>> {
//! # fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
//! let mut feed = hypercore::open("./feed.db")?;
//!
//! feed.append(b"hello")?;
Expand All @@ -27,9 +27,6 @@
//! [Dat]: https://github.com/datrs
//! [Feed]: crate::feed::Feed

#[macro_use]
extern crate failure;

pub mod bitfield;
pub mod prelude;

Expand All @@ -53,12 +50,7 @@ pub use ed25519_dalek::{PublicKey, SecretKey};

use std::path::Path;

use failure::Error;

/// A specialized `Result` type for Hypercore operations.
pub type Result<T> = std::result::Result<T, Error>;

/// Create a new Hypercore `Feed`.
pub fn open<P: AsRef<Path>>(path: P) -> Result<Feed<random_access_disk::RandomAccessDisk>> {
pub fn open<P: AsRef<Path>>(path: P) -> anyhow::Result<Feed<random_access_disk::RandomAccessDisk>> {
Feed::open(path)
}
67 changes: 49 additions & 18 deletions src/storage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@ pub use self::node::Node;
pub use self::persist::Persist;
pub use merkle_tree_stream::Node as NodeTrait;

use crate::Result;
use anyhow::{anyhow, ensure, Result};
use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH};
use failure::Error;
use flat_tree as flat;
use random_access_disk::RandomAccessDisk;
use random_access_memory::RandomAccessMemory;
Expand Down Expand Up @@ -58,7 +57,7 @@ where

impl<T> Storage<T>
where
T: RandomAccess<Error = Error> + Debug,
T: RandomAccess<Error = Box<dyn std::error::Error + Send + Sync>> + Debug,
{
/// Create a new instance. Takes a keypair and a callback to create new
/// storage instances.
Expand All @@ -77,21 +76,30 @@ where
};

let header = create_bitfield();
instance.bitfield.write(0, &header.to_vec())?;
instance
.bitfield
.write(0, &header.to_vec())
.map_err(|e| anyhow!(e))?;

let header = create_signatures();
instance.signatures.write(0, &header.to_vec())?;
instance
.signatures
.write(0, &header.to_vec())
.map_err(|e| anyhow!(e))?;

let header = create_tree();
instance.tree.write(0, &header.to_vec())?;
instance
.tree
.write(0, &header.to_vec())
.map_err(|e| anyhow!(e))?;

Ok(instance)
}

/// Write data to the feed.
#[inline]
pub fn write_data(&mut self, offset: u64, data: &[u8]) -> Result<()> {
self.data.write(offset, &data)
self.data.write(offset, &data).map_err(|e| anyhow!(e))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
self.data.write(offset, &data).map_err(|e| anyhow!(e))
Ok(self.data.write(offset, &data)?)

}

/// Write a byte vector to a data storage (random-access instance) at the
Expand All @@ -114,7 +122,7 @@ where
format!("length `{:?} != {:?}`", range.count(), data.len())
);

self.data.write(range.start, data)
self.data.write(range.start, data).map_err(|e| anyhow!(e))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using ? will map the error for you.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now I remember why I've done this .map_err.

My previous attempt to use ? led me to unsized types:

error[E0277]: the size for values of type `dyn std::error::Error + std::marker::Send + std::marker::Sync` cannot be known at compilation time
  --> src\storage\mod.rs:80:59
   |
80 |         instance.bitfield.write(0, &header.to_vec()).await?;
   |                                                           ^ doesn't have a size known at compile-time
   |
   = help: the trait `std::marker::Sized` is not implemented for `dyn std::error::Error + std::marker::Send + std::marker::Sync`
   = note: to learn more, visit <https://doc.rust-lang.org/book/ch19-04-advanced-types.html#dynamically-sized-types-and-the-sized-trait>
   = note: required because of the requirements on the impl of `std::error::Error` for `std::boxed::Box<dyn std::error::Error + std::marker::Send + std::marker::Sync>`
   = note: required because of the requirements on the impl of `std::convert::From<std::boxed::Box<dyn std::error::Error + std::marker::Send + std::marker::Sync>>` for `anyhow::Error`
   = note: required by `std::convert::From::from`

It seems that anyhow::Error does not implement std::error::Error directly. This issue dtolnay/anyhow#63 led me to attempt calling .into(), which also failed:

error[E0284]: type annotations needed
  --> src\storage\mod.rs:80:60
   |
80 |         instance.bitfield.write(0, &header.to_vec()).await.into()?;
   |         ---------------------------------------------------^^^^--
   |         |                                                  |
   |         |                                                  cannot infer type
   |         this method call resolves to `T`
   |
   = note: cannot resolve `<_ as std::ops::Try>::Ok == _`

Which also didn't work even with a type specification:

error[E0277]: the trait bound `std::result::Result<(), anyhow::Error>: std::convert::From<std::result::Result<(), std::boxed::Box<dyn std::error::Error + std::marker::Send + std::marker::Sync>>>` is not satisfied
  --> src\storage\mod.rs:83:93
   |
83 |         let result: anyhow::Result<()> = instance.bitfield.write(0, &header.to_vec()).await.into();
   |                                                                                             ^^^^ the trait `std::convert::From<std::result::Result<(), std::boxed::Box<dyn std::error::Error + std::marker::Send + std::marker::Sync>>>` is not implemented for `std::result::Result<(), anyhow::Error>`
   |
   = note: required because of the requirements on the impl of `std::convert::Into<std::result::Result<(), anyhow::Error>>` for `std::result::Result<(), std::boxed::Box<dyn std::error::Error + std::marker::Send + std::marker::Sync>>`

There is an upstream issue documenting this conversion error dtolnay/anyhow#66, but I'm not sure if it will fixed, as there is a conflict on the impl of the conversions as documented on the previous discussion.

That led me to .map_err and it compiled without needing to implement a local structure to wrap stderr.

}

/// Get data from disk that the user has written to it. This is stored
Expand All @@ -124,12 +132,18 @@ where
pub fn get_data(&mut self, index: u64) -> Result<Vec<u8>> {
let cached_nodes = Vec::new(); // TODO: reuse allocation.
let range = self.data_offset(index, &cached_nodes)?;
self.data.read(range.start, range.count() as u64)
self.data
.read(range.start, range.count() as u64)
.map_err(|e| anyhow!(e))
}

/// Search the signature stores for a `Signature`, starting at `index`.
pub fn next_signature(&mut self, index: u64) -> Result<Signature> {
let bytes = self.signatures.read(HEADER_OFFSET + 64 * index, 64)?;
let bytes = self
.signatures
.read(HEADER_OFFSET + 64 * index, 64)
.map_err(|e| anyhow!(e))?;

if not_zeroes(&bytes) {
Ok(Signature::from_bytes(&bytes)?)
} else {
Expand All @@ -140,7 +154,10 @@ where
/// Get a `Signature` from the store.
#[inline]
pub fn get_signature(&mut self, index: u64) -> Result<Signature> {
let bytes = self.signatures.read(HEADER_OFFSET + 64 * index, 64)?;
let bytes = self
.signatures
.read(HEADER_OFFSET + 64 * index, 64)
.map_err(|e| anyhow!(e))?;
ensure!(not_zeroes(&bytes), "No signature found");
Ok(Signature::from_bytes(&bytes)?)
}
Expand All @@ -153,6 +170,7 @@ where
let signature = signature.borrow();
self.signatures
.write(HEADER_OFFSET + 64 * index, &signature.to_bytes())
.map_err(|e| anyhow!(e))
}

/// TODO(yw) docs
Expand Down Expand Up @@ -208,7 +226,10 @@ where
/// Get a `Node` from the `tree` storage.
#[inline]
pub fn get_node(&mut self, index: u64) -> Result<Node> {
let buf = self.tree.read(HEADER_OFFSET + 40 * index, 40)?;
let buf = self
.tree
.read(HEADER_OFFSET + 40 * index, 40)
.map_err(|e| anyhow!(e))?;
let node = Node::from_bytes(index, &buf)?;
Ok(node)
}
Expand All @@ -220,20 +241,27 @@ where
pub fn put_node(&mut self, node: &Node) -> Result<()> {
let index = node.index();
let buf = node.to_bytes()?;
self.tree.write(HEADER_OFFSET + 40 * index, &buf)
self.tree
.write(HEADER_OFFSET + 40 * index, &buf)
.map_err(|e| anyhow!(e))
}

/// Write data to the internal bitfield module.
/// TODO: Ensure the chunk size is correct.
/// NOTE: Should we create a bitfield entry type?
#[inline]
pub fn put_bitfield(&mut self, offset: u64, data: &[u8]) -> Result<()> {
self.bitfield.write(HEADER_OFFSET + offset, data)
self.bitfield
.write(HEADER_OFFSET + offset, data)
.map_err(|e| anyhow!(e))
}

/// Read a public key from storage
pub fn read_public_key(&mut self) -> Result<PublicKey> {
let buf = self.keypair.read(0, PUBLIC_KEY_LENGTH as u64)?;
let buf = self
.keypair
.read(0, PUBLIC_KEY_LENGTH as u64)
.map_err(|e| anyhow!(e))?;
let public_key = PublicKey::from_bytes(&buf)?;
Ok(public_key)
}
Expand All @@ -242,21 +270,24 @@ where
pub fn read_secret_key(&mut self) -> Result<SecretKey> {
let buf = self
.keypair
.read(PUBLIC_KEY_LENGTH as u64, SECRET_KEY_LENGTH as u64)?;
.read(PUBLIC_KEY_LENGTH as u64, SECRET_KEY_LENGTH as u64)
.map_err(|e| anyhow!(e))?;
let secret_key = SecretKey::from_bytes(&buf)?;
Ok(secret_key)
}

/// Write a public key to the storage
pub fn write_public_key(&mut self, public_key: &PublicKey) -> Result<()> {
let buf: [u8; PUBLIC_KEY_LENGTH] = public_key.to_bytes();
self.keypair.write(0, &buf)
self.keypair.write(0, &buf).map_err(|e| anyhow!(e))
}

/// Write a secret key to the storage
pub fn write_secret_key(&mut self, secret_key: &SecretKey) -> Result<()> {
let buf: [u8; SECRET_KEY_LENGTH] = secret_key.to_bytes();
self.keypair.write(PUBLIC_KEY_LENGTH as u64, &buf)
self.keypair
.write(PUBLIC_KEY_LENGTH as u64, &buf)
.map_err(|e| anyhow!(e))
}

/// Tries to read a partial keypair (ie: with an optional secret_key) from the storage
Expand Down
Loading