diff --git a/crates/compression-codecs/src/brotli/decoder.rs b/crates/compression-codecs/src/brotli/decoder.rs index 21e57f3..248a996 100644 --- a/crates/compression-codecs/src/brotli/decoder.rs +++ b/crates/compression-codecs/src/brotli/decoder.rs @@ -1,37 +1,45 @@ -use crate::Decode; -use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult, BrotliState}; -use compression_core::util::PartialBuffer; +use crate::DecodeV2; +use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{fmt, io}; +type BrotliState = brotli::BrotliState; + pub struct BrotliDecoder { // `BrotliState` is very large (over 2kb) which is why we're boxing it. - state: Box>, + state: Box, } impl Default for BrotliDecoder { fn default() -> Self { Self { - state: Box::new(BrotliState::new( - StandardAlloc::default(), - StandardAlloc::default(), - StandardAlloc::default(), - )), + state: Box::new(Self::new_brotli_state()), } } } impl BrotliDecoder { + fn new_brotli_state() -> BrotliState { + BrotliState::new( + StandardAlloc::default(), + StandardAlloc::default(), + StandardAlloc::default(), + ) + } + pub fn new() -> Self { Self::default() } fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { + output.initialize_unwritten(); + let in_buf = input.unwritten(); - let out_buf = output.unwritten_mut(); + let out_buf = output.unwritten_initialized_mut(); let mut input_len = 0; let mut output_len = 0; @@ -57,20 +65,16 @@ impl BrotliDecoder { } } -impl Decode for BrotliDecoder { +impl DecodeV2 for BrotliDecoder { fn reinit(&mut self) -> io::Result<()> { - self.state = Box::new(BrotliState::new( - StandardAlloc::default(), - StandardAlloc::default(), - StandardAlloc::default(), - )); + *self.state = Self::new_brotli_state(); Ok(()) } fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { match self.decode(input, output)? { BrotliResult::ResultSuccess => Ok(true), @@ -79,10 +83,7 @@ impl Decode for BrotliDecoder { } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { match self.decode(&mut PartialBuffer::new(&[][..]), output)? { BrotliResult::ResultSuccess | BrotliResult::NeedsMoreInput => Ok(true), BrotliResult::NeedsMoreOutput => Ok(false), @@ -90,10 +91,7 @@ impl Decode for BrotliDecoder { } } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { match self.decode(&mut PartialBuffer::new(&[][..]), output)? { BrotliResult::ResultSuccess => Ok(true), BrotliResult::NeedsMoreOutput => Ok(false), diff --git a/crates/compression-codecs/src/brotli/encoder.rs b/crates/compression-codecs/src/brotli/encoder.rs index c14dd33..bdec594 100644 --- a/crates/compression-codecs/src/brotli/encoder.rs +++ b/crates/compression-codecs/src/brotli/encoder.rs @@ -1,10 +1,10 @@ -use crate::{brotli::params::EncoderParams, Encode}; +use crate::{brotli::params::EncoderParams, EncodeV2}; use brotli::enc::{ backward_references::BrotliEncoderParams, encode::{BrotliEncoderOperation, BrotliEncoderStateStruct}, StandardAlloc, }; -use compression_core::util::PartialBuffer; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{fmt, io}; pub struct BrotliEncoder { @@ -21,12 +21,14 @@ impl BrotliEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, op: BrotliEncoderOperation, ) -> io::Result<()> { + output.initialize_unwritten(); + let in_buf = input.unwritten(); - let out_buf = output.unwritten_mut(); + let out_buf = output.unwritten_initialized_mut(); let mut input_len = 0; let mut output_len = 0; @@ -52,11 +54,11 @@ impl BrotliEncoder { } } -impl Encode for BrotliEncoder { +impl EncodeV2 for BrotliEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result<()> { self.encode( input, @@ -65,10 +67,7 @@ impl Encode for BrotliEncoder { ) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { self.encode( &mut PartialBuffer::new(&[][..]), output, @@ -78,10 +77,7 @@ impl Encode for BrotliEncoder { Ok(!self.state.has_more_output()) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { self.encode( &mut PartialBuffer::new(&[][..]), output, diff --git a/crates/compression-codecs/src/bzip2/decoder.rs b/crates/compression-codecs/src/bzip2/decoder.rs index 7b99338..537549e 100644 --- a/crates/compression-codecs/src/bzip2/decoder.rs +++ b/crates/compression-codecs/src/bzip2/decoder.rs @@ -1,6 +1,6 @@ -use crate::Decode; +use crate::DecodeV2; use bzip2::{Decompress, Status}; -use compression_core::util::PartialBuffer; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{fmt, io}; pub struct BzDecoder { @@ -33,15 +33,17 @@ impl BzDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { + output.initialize_unwritten(); + let prior_in = self.decompress.total_in(); let prior_out = self.decompress.total_out(); let status = self .decompress - .decompress(input.unwritten(), output.unwritten_mut()) + .decompress(input.unwritten(), output.unwritten_initialized_mut()) .map_err(io::Error::other)?; input.advance((self.decompress.total_in() - prior_in) as usize); @@ -51,7 +53,7 @@ impl BzDecoder { } } -impl Decode for BzDecoder { +impl DecodeV2 for BzDecoder { fn reinit(&mut self) -> io::Result<()> { self.decompress = Decompress::new(false); Ok(()) @@ -59,8 +61,8 @@ impl Decode for BzDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { match self.decode(input, output)? { // Decompression went fine, nothing much to report. @@ -84,27 +86,21 @@ impl Decode for BzDecoder { } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { self.decode(&mut PartialBuffer::new(&[][..]), output)?; loop { - let old_len = output.written().len(); + let old_len = output.written_len(); self.decode(&mut PartialBuffer::new(&[][..]), output)?; - if output.written().len() == old_len { + if output.written_len() == old_len { break; } } - Ok(!output.unwritten().is_empty()) + Ok(!output.has_no_spare_space()) } - fn finish( - &mut self, - _output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, _output: &mut WriteBuffer<'_>) -> io::Result { Ok(true) } } diff --git a/crates/compression-codecs/src/bzip2/encoder.rs b/crates/compression-codecs/src/bzip2/encoder.rs index 66650e7..b9a5d80 100644 --- a/crates/compression-codecs/src/bzip2/encoder.rs +++ b/crates/compression-codecs/src/bzip2/encoder.rs @@ -1,6 +1,6 @@ -use crate::{bzip2::params::Bzip2EncoderParams, Encode}; +use crate::{bzip2::params::Bzip2EncoderParams, EncodeV2}; use bzip2::{Action, Compress, Compression, Status}; -use compression_core::util::PartialBuffer; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{fmt, io}; pub struct BzEncoder { @@ -48,16 +48,22 @@ impl BzEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, action: Action, ) -> io::Result { + output.initialize_unwritten(); + let prior_in = self.compress.total_in(); let prior_out = self.compress.total_out(); let status = self .compress - .compress(input.unwritten(), output.unwritten_mut(), action) + .compress( + input.unwritten(), + output.unwritten_initialized_mut(), + action, + ) .map_err(io::Error::other)?; input.advance((self.compress.total_in() - prior_in) as usize); @@ -67,11 +73,11 @@ impl BzEncoder { } } -impl Encode for BzEncoder { +impl EncodeV2 for BzEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result<()> { match self.encode(input, output, Action::Run)? { // Decompression went fine, nothing much to report. @@ -95,10 +101,7 @@ impl Encode for BzEncoder { } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Flush)? { // Decompression went fine, nothing much to report. Status::Ok => unreachable!(), @@ -121,10 +124,7 @@ impl Encode for BzEncoder { } } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { match self.encode(&mut PartialBuffer::new(&[][..]), output, Action::Finish)? { // Decompression went fine, nothing much to report. Status::Ok => Ok(false), diff --git a/crates/compression-codecs/src/deflate/decoder.rs b/crates/compression-codecs/src/deflate/decoder.rs index d945241..e7f88ce 100644 --- a/crates/compression-codecs/src/deflate/decoder.rs +++ b/crates/compression-codecs/src/deflate/decoder.rs @@ -1,5 +1,5 @@ -use crate::{Decode, FlateDecoder}; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, FlateDecoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::io::Result; #[derive(Debug)] @@ -21,7 +21,7 @@ impl DeflateDecoder { } } -impl Decode for DeflateDecoder { +impl DecodeV2 for DeflateDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit()?; Ok(()) @@ -29,23 +29,17 @@ impl Decode for DeflateDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { self.inner.decode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/deflate/encoder.rs b/crates/compression-codecs/src/deflate/encoder.rs index f24212b..48b3d9c 100644 --- a/crates/compression-codecs/src/deflate/encoder.rs +++ b/crates/compression-codecs/src/deflate/encoder.rs @@ -1,5 +1,5 @@ -use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; -use compression_core::util::PartialBuffer; +use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::io::Result; #[derive(Debug)] @@ -19,26 +19,20 @@ impl DeflateEncoder { } } -impl Encode for DeflateEncoder { +impl EncodeV2 for DeflateEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { self.inner.encode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/deflate64/decoder.rs b/crates/compression-codecs/src/deflate64/decoder.rs index 1bdf425..39869a6 100644 --- a/crates/compression-codecs/src/deflate64/decoder.rs +++ b/crates/compression-codecs/src/deflate64/decoder.rs @@ -1,5 +1,5 @@ -use crate::Decode; -use compression_core::util::PartialBuffer; +use crate::DecodeV2; +use compression_core::util::{PartialBuffer, WriteBuffer}; use deflate64::InflaterManaged; use std::io::{Error, ErrorKind, Result}; @@ -23,12 +23,14 @@ impl Deflate64Decoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { + output.initialize_unwritten(); + let result = self .inflater - .inflate(input.unwritten(), output.unwritten_mut()); + .inflate(input.unwritten(), output.unwritten_initialized_mut()); input.advance(result.bytes_consumed); output.advance(result.bytes_written); @@ -41,41 +43,35 @@ impl Deflate64Decoder { } } -impl Decode for Deflate64Decoder { +impl DecodeV2 for Deflate64Decoder { fn reinit(&mut self) -> Result<()> { - self.inflater = Box::new(InflaterManaged::new()); + *self.inflater = InflaterManaged::new(); Ok(()) } fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { self.decode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - self.decode(&mut PartialBuffer::new([]), output)?; + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.decode(&mut PartialBuffer::new(&[]), output)?; loop { - let old_len = output.written().len(); - self.decode(&mut PartialBuffer::new([]), output)?; - if output.written().len() == old_len { + let old_len = output.written_len(); + self.decode(&mut PartialBuffer::new(&[]), output)?; + if output.written_len() == old_len { break; } } - Ok(!output.unwritten().is_empty()) + Ok(!output.has_no_spare_space()) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - self.decode(&mut PartialBuffer::new([]), output) + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.decode(&mut PartialBuffer::new(&[]), output) } } diff --git a/crates/compression-codecs/src/flate/decoder.rs b/crates/compression-codecs/src/flate/decoder.rs index 8797ff1..2424daa 100644 --- a/crates/compression-codecs/src/flate/decoder.rs +++ b/crates/compression-codecs/src/flate/decoder.rs @@ -1,5 +1,5 @@ -use crate::Decode; -use compression_core::util::PartialBuffer; +use crate::DecodeV2; +use compression_core::util::{PartialBuffer, WriteBuffer}; use flate2::{Decompress, FlushDecompress, Status}; use std::io; @@ -19,16 +19,20 @@ impl FlateDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, flush: FlushDecompress, ) -> io::Result { + output.initialize_unwritten(); + let prior_in = self.decompress.total_in(); let prior_out = self.decompress.total_out(); - let status = - self.decompress - .decompress(input.unwritten(), output.unwritten_mut(), flush)?; + let status = self.decompress.decompress( + input.unwritten(), + output.unwritten_initialized_mut(), + flush, + )?; input.advance((self.decompress.total_in() - prior_in) as usize); output.advance((self.decompress.total_out() - prior_out) as usize); @@ -37,7 +41,7 @@ impl FlateDecoder { } } -impl Decode for FlateDecoder { +impl DecodeV2 for FlateDecoder { fn reinit(&mut self) -> io::Result<()> { self.decompress.reset(self.zlib_header); Ok(()) @@ -45,8 +49,8 @@ impl Decode for FlateDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { match self.decode(input, output, FlushDecompress::None)? { Status::Ok => Ok(false), @@ -55,10 +59,7 @@ impl Decode for FlateDecoder { } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { self.decode( &mut PartialBuffer::new(&[][..]), output, @@ -66,24 +67,21 @@ impl Decode for FlateDecoder { )?; loop { - let old_len = output.written().len(); + let old_len = output.written_len(); self.decode( &mut PartialBuffer::new(&[][..]), output, FlushDecompress::None, )?; - if output.written().len() == old_len { + if output.written_len() == old_len { break; } } - Ok(!output.unwritten().is_empty()) + Ok(!output.has_no_spare_space()) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { match self.decode( &mut PartialBuffer::new(&[][..]), output, diff --git a/crates/compression-codecs/src/flate/encoder.rs b/crates/compression-codecs/src/flate/encoder.rs index 3622305..db15324 100644 --- a/crates/compression-codecs/src/flate/encoder.rs +++ b/crates/compression-codecs/src/flate/encoder.rs @@ -1,5 +1,5 @@ -use crate::{flate::params::FlateEncoderParams, Encode}; -use compression_core::util::PartialBuffer; +use crate::{flate::params::FlateEncoderParams, EncodeV2}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use flate2::{Compress, FlushCompress, Status}; use std::io; @@ -24,16 +24,18 @@ impl FlateEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, flush: FlushCompress, ) -> io::Result { + output.initialize_unwritten(); + let prior_in = self.compress.total_in(); let prior_out = self.compress.total_out(); - let status = self - .compress - .compress(input.unwritten(), output.unwritten_mut(), flush)?; + let status = + self.compress + .compress(input.unwritten(), output.unwritten_initialized_mut(), flush)?; input.advance((self.compress.total_in() - prior_in) as usize); output.advance((self.compress.total_out() - prior_out) as usize); @@ -42,11 +44,11 @@ impl FlateEncoder { } } -impl Encode for FlateEncoder { +impl EncodeV2 for FlateEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result<()> { self.flushed = false; match self.encode(input, output, FlushCompress::None)? { @@ -56,10 +58,7 @@ impl Encode for FlateEncoder { } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { // We need to keep track of whether we've already flushed otherwise we'll just keep writing // out sync blocks continuously and probably never complete flushing. if self.flushed { @@ -73,26 +72,23 @@ impl Encode for FlateEncoder { )?; loop { - let old_len = output.written().len(); + let old_len = output.written_len(); self.encode( &mut PartialBuffer::new(&[][..]), output, FlushCompress::None, )?; - if output.written().len() == old_len { + if output.written_len() == old_len { break; } } - let internal_flushed = !output.unwritten().is_empty(); + let internal_flushed = !output.has_no_spare_space(); self.flushed = internal_flushed; Ok(internal_flushed) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { self.flushed = false; match self.encode( &mut PartialBuffer::new(&[][..]), diff --git a/crates/compression-codecs/src/gzip/decoder.rs b/crates/compression-codecs/src/gzip/decoder.rs index 2e20262..5a72b04 100644 --- a/crates/compression-codecs/src/gzip/decoder.rs +++ b/crates/compression-codecs/src/gzip/decoder.rs @@ -1,6 +1,6 @@ use super::header::{self, Header}; -use crate::{Decode, FlateDecoder}; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, FlateDecoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use flate2::Crc; use std::io::{Error, ErrorKind, Result}; @@ -64,11 +64,11 @@ impl GzipDecoder { Self::default() } - fn process, O: AsRef<[u8]> + AsMut<[u8]>>( + fn process( &mut self, - input: &mut PartialBuffer, - output: &mut PartialBuffer, - inner: impl Fn(&mut Self, &mut PartialBuffer, &mut PartialBuffer) -> Result, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, + inner: impl Fn(&mut Self, &mut PartialBuffer<&[u8]>, &mut WriteBuffer<'_>) -> Result, ) -> Result { loop { match &mut self.state { @@ -80,11 +80,11 @@ impl GzipDecoder { } State::Decoding => { - let prior = output.written().len(); + let prior = output.written_len(); let res = inner(self, input, output); - if output.written().len() > prior { + if output.written_len() > prior { // update CRC even if there was an error self.crc.update(&output.written()[prior..]); } @@ -112,14 +112,14 @@ impl GzipDecoder { return Ok(true); } - if input.unwritten().is_empty() || output.unwritten().is_empty() { + if input.unwritten().is_empty() || output.has_no_spare_space() { return Ok(false); } } } } -impl Decode for GzipDecoder { +impl DecodeV2 for GzipDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit()?; self.crc = Crc::new(); @@ -130,24 +130,21 @@ impl Decode for GzipDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { self.process(input, output, |this, input, output| { this.inner.decode(input, output) }) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { loop { match self.state { State::Header(_) | State::Footer(_) | State::Done => return Ok(true), State::Decoding => { - let prior = output.written().len(); + let prior = output.written_len(); let done = self.inner.flush(output)?; self.crc.update(&output.written()[prior..]); if done { @@ -156,24 +153,18 @@ impl Decode for GzipDecoder { } }; - if output.unwritten().is_empty() { + if output.has_no_spare_space() { return Ok(false); } } } - fn finish( - &mut self, - _output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, _output: &mut WriteBuffer<'_>) -> Result { // Because of the footer we have to have already flushed all the data out before we get here if let State::Done = self.state { Ok(true) } else { - Err(Error::new( - ErrorKind::UnexpectedEof, - "unexpected end of file", - )) + Err(Error::from(ErrorKind::UnexpectedEof)) } } } diff --git a/crates/compression-codecs/src/gzip/encoder.rs b/crates/compression-codecs/src/gzip/encoder.rs index b0d152a..9b706d8 100644 --- a/crates/compression-codecs/src/gzip/encoder.rs +++ b/crates/compression-codecs/src/gzip/encoder.rs @@ -1,5 +1,5 @@ -use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; -use compression_core::util::PartialBuffer; +use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use flate2::{Compression, Crc}; use std::io; @@ -49,11 +49,11 @@ impl GzipEncoder { } } -impl Encode for GzipEncoder { +impl EncodeV2 for GzipEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result<()> { loop { match &mut self.state { @@ -76,16 +76,13 @@ impl Encode for GzipEncoder { } }; - if input.unwritten().is_empty() || output.unwritten().is_empty() { + if input.unwritten().is_empty() || output.has_no_spare_space() { return Ok(()); } } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { loop { let done = match &mut self.state { State::Header(header) => { @@ -117,16 +114,13 @@ impl Encode for GzipEncoder { return Ok(true); } - if output.unwritten().is_empty() { + if output.has_no_spare_space() { return Ok(false); } } } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { loop { match &mut self.state { State::Header(header) => { @@ -158,7 +152,7 @@ impl Encode for GzipEncoder { return Ok(true); } - if output.unwritten().is_empty() { + if output.has_no_spare_space() { return Ok(false); } } diff --git a/crates/compression-codecs/src/gzip/header.rs b/crates/compression-codecs/src/gzip/header.rs index abd98bf..459fcb3 100644 --- a/crates/compression-codecs/src/gzip/header.rs +++ b/crates/compression-codecs/src/gzip/header.rs @@ -62,10 +62,7 @@ impl Header { } impl Parser { - pub(super) fn input( - &mut self, - input: &mut PartialBuffer>, - ) -> io::Result> { + pub(super) fn input(&mut self, input: &mut PartialBuffer<&[u8]>) -> io::Result> { loop { match &mut self.state { State::Fixed(data) => { diff --git a/crates/compression-codecs/src/lib.rs b/crates/compression-codecs/src/lib.rs index 1ad3ec1..32c186a 100644 --- a/crates/compression-codecs/src/lib.rs +++ b/crates/compression-codecs/src/lib.rs @@ -31,7 +31,7 @@ pub mod zlib; #[cfg(feature = "zstd")] pub mod zstd; -use compression_core::util::PartialBuffer; +use compression_core::util::{PartialBuffer, WriteBuffer}; #[cfg(feature = "brotli")] pub use self::brotli::{BrotliDecoder, BrotliEncoder}; @@ -58,6 +58,39 @@ pub use self::zlib::{ZlibDecoder, ZlibEncoder}; #[cfg(feature = "zstd")] pub use self::zstd::{ZstdDecoder, ZstdEncoder}; +fn forward_output( + output: &mut PartialBuffer + AsMut<[u8]>>, + f: impl FnOnce(&mut WriteBuffer<'_>) -> R, +) -> R { + let written_len = output.written_len(); + + let output_buffer = output.get_mut(); + let mut write_buffer = WriteBuffer::new_initialized(output_buffer.as_mut()); + write_buffer.advance(written_len); + + let result = f(&mut write_buffer); + let new_written_len = write_buffer.written_len(); + output.advance(new_written_len - written_len); + result +} + +fn forward_input_output( + input: &mut PartialBuffer>, + output: &mut PartialBuffer + AsMut<[u8]>>, + f: impl FnOnce(&mut PartialBuffer<&[u8]>, &mut WriteBuffer<'_>) -> R, +) -> R { + let written_len = input.written_len(); + + let input_buffer = input.get_mut(); + let mut partial_buffer = PartialBuffer::new(input_buffer.as_ref()); + partial_buffer.advance(written_len); + + let result = forward_output(output, |output| f(&mut partial_buffer, output)); + let new_written_len = partial_buffer.written_len(); + input.advance(new_written_len - written_len); + result +} + pub trait Encode { fn encode( &mut self, @@ -75,6 +108,50 @@ pub trait Encode { output: &mut PartialBuffer + AsMut<[u8]>>, ) -> Result; } +impl Encode for T { + fn encode( + &mut self, + input: &mut PartialBuffer>, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result<()> { + forward_input_output(input, output, |input, output| { + EncodeV2::encode(self, input, output) + }) + } + + fn flush( + &mut self, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result { + forward_output(output, |output| EncodeV2::flush(self, output)) + } + + fn finish( + &mut self, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result { + forward_output(output, |output| EncodeV2::finish(self, output)) + } +} + +/// version 2 of [`Encode`] that is trait object safe. +/// +/// The different from [`Encode`] is that: +/// - It doesn't have any generic in it, so it is trait object safe +/// - It uses [`WriteBuffer`] for output, which will support uninitialized buffer. +pub trait EncodeV2 { + fn encode( + &mut self, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, + ) -> Result<()>; + + /// Returns whether the internal buffers are flushed + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result; + + /// Returns whether the internal buffers are flushed and the end of the stream is written + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result; +} pub trait Decode { /// Reinitializes this decoder ready to decode a new member/frame of data. @@ -98,6 +175,59 @@ pub trait Decode { ) -> Result; } +impl Decode for T { + fn reinit(&mut self) -> Result<()> { + DecodeV2::reinit(self) + } + + fn decode( + &mut self, + input: &mut PartialBuffer>, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result { + forward_input_output(input, output, |input, output| { + DecodeV2::decode(self, input, output) + }) + } + + fn flush( + &mut self, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result { + forward_output(output, |output| DecodeV2::flush(self, output)) + } + + fn finish( + &mut self, + output: &mut PartialBuffer + AsMut<[u8]>>, + ) -> Result { + forward_output(output, |output| DecodeV2::finish(self, output)) + } +} + +/// version 2 [`Decode`] that is trait object safe. +/// +/// The different from [`Decode`] is that: +/// - It doesn't have any generic in it, so it is trait object safe +/// - It uses [`WriteBuffer`] for output, which will support uninitialized buffer. +pub trait DecodeV2 { + /// Reinitializes this decoder ready to decode a new member/frame of data. + fn reinit(&mut self) -> Result<()>; + + /// Returns whether the end of the stream has been read + fn decode( + &mut self, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, + ) -> Result; + + /// Returns whether the internal buffers are flushed + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result; + + /// Returns whether the internal buffers are flushed + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result; +} + pub trait DecodedSize { /// Returns the size of the input when uncompressed. fn decoded_size(input: &[u8]) -> Result; diff --git a/crates/compression-codecs/src/lz4/decoder.rs b/crates/compression-codecs/src/lz4/decoder.rs index 185d7a6..aa7c71d 100644 --- a/crates/compression-codecs/src/lz4/decoder.rs +++ b/crates/compression-codecs/src/lz4/decoder.rs @@ -1,5 +1,8 @@ -use crate::Decode; -use compression_core::{unshared::Unshared, util::PartialBuffer}; +use crate::DecodeV2; +use compression_core::{ + unshared::Unshared, + util::{PartialBuffer, WriteBuffer}, +}; use lz4::liblz4::{ check_error, LZ4FDecompressionContext, LZ4F_createDecompressionContext, LZ4F_decompress, LZ4F_freeDecompressionContext, LZ4F_resetDecompressionContext, LZ4F_VERSION, @@ -44,7 +47,7 @@ impl Lz4Decoder { } } -impl Decode for Lz4Decoder { +impl DecodeV2 for Lz4Decoder { fn reinit(&mut self) -> Result<()> { unsafe { LZ4F_resetDecompressionContext(self.ctx.get_mut().ctx) }; Ok(()) @@ -52,15 +55,17 @@ impl Decode for Lz4Decoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { - let mut output_size = output.unwritten().len(); + output.initialize_unwritten(); + + let mut output_size = output.unwritten_initialized_mut().len(); let mut input_size = input.unwritten().len(); let remaining = unsafe { check_error(LZ4F_decompress( self.ctx.get_mut().ctx, - output.unwritten_mut().as_mut_ptr(), + output.unwritten_initialized_mut().as_mut_ptr(), &mut output_size, input.unwritten().as_ptr(), &mut input_size, @@ -72,27 +77,21 @@ impl Decode for Lz4Decoder { Ok(remaining == 0) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.decode(&mut PartialBuffer::new(&[][..]), output)?; loop { - let old_len = output.written().len(); + let old_len = output.written_len(); self.decode(&mut PartialBuffer::new(&[][..]), output)?; - if output.written().len() == old_len { + if output.written_len() == old_len { break; } } - Ok(!output.unwritten().is_empty()) + Ok(!output.has_no_spare_space()) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.flush(output) } } diff --git a/crates/compression-codecs/src/lz4/encoder.rs b/crates/compression-codecs/src/lz4/encoder.rs index d306e02..bf070d5 100644 --- a/crates/compression-codecs/src/lz4/encoder.rs +++ b/crates/compression-codecs/src/lz4/encoder.rs @@ -1,5 +1,8 @@ -use crate::{lz4::params::EncoderParams, Encode}; -use compression_core::{unshared::Unshared, util::PartialBuffer}; +use crate::{lz4::params::EncoderParams, EncodeV2}; +use compression_core::{ + unshared::Unshared, + util::{PartialBuffer, WriteBuffer}, +}; use lz4::liblz4::{ check_error, LZ4FCompressionContext, LZ4FPreferences, LZ4F_compressBegin, LZ4F_compressBound, LZ4F_compressEnd, LZ4F_compressUpdate, LZ4F_createCompressionContext, LZ4F_flush, @@ -23,12 +26,11 @@ enum State { Done, } -enum Lz4Fn<'a, T> -where - T: AsRef<[u8]>, -{ +enum Lz4Fn<'a, 'b> { Begin, - Update { input: &'a mut PartialBuffer }, + Update { + input: &'a mut PartialBuffer<&'b [u8]>, + }, Flush, End, } @@ -88,10 +90,7 @@ impl Lz4Encoder { self.block_buffer_size } - fn drain_buffer( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> (usize, usize) { + fn drain_buffer(&mut self, output: &mut WriteBuffer<'_>) -> (usize, usize) { match self.maybe_buffer.as_mut() { Some(buffer) => { let drained_bytes = output.copy_unwritten_from(buffer); @@ -101,14 +100,7 @@ impl Lz4Encoder { } } - fn write<'a, T>( - &'a mut self, - lz4_fn: Lz4Fn<'a, T>, - output: &'a mut PartialBuffer + AsMut<[u8]>>, - ) -> Result - where - T: AsRef<[u8]>, - { + fn write(&mut self, lz4_fn: Lz4Fn<'_, '_>, output: &mut WriteBuffer<'_>) -> Result { let (drained_before, undrained) = self.drain_buffer(output); if undrained > 0 { return Ok(drained_before); @@ -125,7 +117,8 @@ impl Lz4Encoder { Lz4Fn::Flush | Lz4Fn::End => self.flush_buffer_size, }; - let output_len = output.unwritten().len(); + output.initialize_unwritten(); + let output_len = output.unwritten_initialized_mut().len(); let (dst_buffer, dst_size, maybe_internal_buffer) = if min_dst_size > output_len { let buffer_size = self.block_buffer_size; @@ -139,7 +132,11 @@ impl Lz4Encoder { Some(buffer), ) } else { - (output.unwritten_mut().as_mut_ptr(), output_len, None) + ( + output.unwritten_initialized_mut().as_mut_ptr(), + output_len, + None, + ) }; let len = match lz4_fn { @@ -206,16 +203,16 @@ impl Lz4Encoder { } } -impl Encode for Lz4Encoder { +impl EncodeV2 for Lz4Encoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { loop { match self.state { State::Header => { - self.write(Lz4Fn::Begin::<&[u8]>, output)?; + self.write(Lz4Fn::Begin, output)?; } State::Encoding => { @@ -227,25 +224,22 @@ impl Encode for Lz4Encoder { } } - if input.unwritten().is_empty() || output.unwritten().is_empty() { + if input.unwritten().is_empty() || output.has_no_spare_space() { return Ok(()); } } } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { loop { let done = match self.state { State::Header => { - self.write(Lz4Fn::Begin::<&[u8]>, output)?; + self.write(Lz4Fn::Begin, output)?; false } State::Encoding => { - let len = self.write(Lz4Fn::Flush::<&[u8]>, output)?; + let len = self.write(Lz4Fn::Flush, output)?; len == 0 } @@ -266,24 +260,21 @@ impl Encode for Lz4Encoder { return Ok(true); } - if output.unwritten().is_empty() { + if output.has_no_spare_space() { return Ok(false); } } } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { loop { match self.state { State::Header => { - self.write(Lz4Fn::Begin::<&[u8]>, output)?; + self.write(Lz4Fn::Begin, output)?; } State::Encoding => { - self.write(Lz4Fn::End::<&[u8]>, output)?; + self.write(Lz4Fn::End, output)?; } State::Footer => { @@ -300,7 +291,7 @@ impl Encode for Lz4Encoder { return Ok(true); } - if output.unwritten().is_empty() { + if output.has_no_spare_space() { return Ok(false); } } diff --git a/crates/compression-codecs/src/lzma/decoder.rs b/crates/compression-codecs/src/lzma/decoder.rs index 4291c45..8e18cca 100644 --- a/crates/compression-codecs/src/lzma/decoder.rs +++ b/crates/compression-codecs/src/lzma/decoder.rs @@ -1,5 +1,5 @@ -use crate::{Decode, DecodedSize, Xz2Decoder}; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, DecodedSize, Xz2Decoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{convert::TryInto, io::Result}; /// Lzma decoding stream @@ -34,30 +34,24 @@ impl LzmaDecoder { } } -impl Decode for LzmaDecoder { +impl DecodeV2 for LzmaDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit() } fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { self.inner.decode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/lzma/encoder.rs b/crates/compression-codecs/src/lzma/encoder.rs index b83a6b6..4ad2286 100644 --- a/crates/compression-codecs/src/lzma/encoder.rs +++ b/crates/compression-codecs/src/lzma/encoder.rs @@ -1,5 +1,8 @@ -use crate::{Encode, Xz2Encoder, Xz2FileFormat}; -use compression_core::{util::PartialBuffer, Level}; +use crate::{EncodeV2, Xz2Encoder, Xz2FileFormat}; +use compression_core::{ + util::{PartialBuffer, WriteBuffer}, + Level, +}; use std::io::Result; /// Lzma encoding stream @@ -22,27 +25,21 @@ impl From for LzmaEncoder { } } -impl Encode for LzmaEncoder { +impl EncodeV2 for LzmaEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { self.inner.encode(input, output) } - fn flush( - &mut self, - _output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, _output: &mut WriteBuffer<'_>) -> Result { // Flush on LZMA 1 is not supported Ok(true) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/xz/decoder.rs b/crates/compression-codecs/src/xz/decoder.rs index 4ff55b0..214143d 100644 --- a/crates/compression-codecs/src/xz/decoder.rs +++ b/crates/compression-codecs/src/xz/decoder.rs @@ -1,5 +1,5 @@ -use crate::{Decode, DecodedSize, Xz2Decoder}; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, DecodedSize, Xz2Decoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::{ convert::TryInto, io::{Error, ErrorKind, Result}, @@ -42,7 +42,7 @@ impl XzDecoder { } } -impl Decode for XzDecoder { +impl DecodeV2 for XzDecoder { fn reinit(&mut self) -> Result<()> { self.skip_padding = Some(4); self.inner.reinit() @@ -50,8 +50,8 @@ impl Decode for XzDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { if let Some(ref mut count) = self.skip_padding { while input.unwritten().first() == Some(&0) { @@ -77,20 +77,14 @@ impl Decode for XzDecoder { self.inner.decode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { if self.skip_padding.is_some() { return Ok(true); } self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { if self.skip_padding.is_some() { return Ok(true); } diff --git a/crates/compression-codecs/src/xz/encoder.rs b/crates/compression-codecs/src/xz/encoder.rs index 5af99ee..e629983 100644 --- a/crates/compression-codecs/src/xz/encoder.rs +++ b/crates/compression-codecs/src/xz/encoder.rs @@ -1,5 +1,8 @@ -use crate::{Encode, Xz2Encoder, Xz2FileFormat}; -use compression_core::{util::PartialBuffer, Level}; +use crate::{EncodeV2, Xz2Encoder, Xz2FileFormat}; +use compression_core::{ + util::{PartialBuffer, WriteBuffer}, + Level, +}; use std::io::Result; /// Xz encoding stream @@ -23,26 +26,20 @@ impl XzEncoder { } } -impl Encode for XzEncoder { +impl EncodeV2 for XzEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { self.inner.encode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/xz2/decoder.rs b/crates/compression-codecs/src/xz2/decoder.rs index 98d9ab8..167c20a 100644 --- a/crates/compression-codecs/src/xz2/decoder.rs +++ b/crates/compression-codecs/src/xz2/decoder.rs @@ -1,5 +1,5 @@ -use crate::{lzma::params::LzmaDecoderParams, Decode, DecodedSize}; -use compression_core::util::PartialBuffer; +use crate::{lzma::params::LzmaDecoderParams, DecodeV2, DecodedSize}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use liblzma::stream::{Action, Status, Stream}; use std::{ convert::TryFrom, @@ -52,27 +52,21 @@ impl Xz2Decoder { Self::try_from(params).unwrap() } -} -impl Decode for Xz2Decoder { - fn reinit(&mut self) -> io::Result<()> { - *self = Self::try_from(self.params.clone())?; - Ok(()) - } - - fn decode( + /// Return `Ok(true)` on stream ends. + fn process( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &[u8], + output: &mut WriteBuffer<'_>, + action: Action, ) -> io::Result { - let previous_in = self.stream.total_in() as usize; let previous_out = self.stream.total_out() as usize; + output.initialize_unwritten(); let status = self .stream - .process(input.unwritten(), output.unwritten_mut(), Action::Run)?; + .process(input, output.unwritten_initialized_mut(), action)?; - input.advance(self.stream.total_in() as usize - previous_in); output.advance(self.stream.total_out() as usize - previous_out); match status { @@ -82,33 +76,34 @@ impl Decode for Xz2Decoder { Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()), } } +} - fn flush( - &mut self, - _output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { - // While decoding flush is a noop - Ok(true) +impl DecodeV2 for Xz2Decoder { + fn reinit(&mut self) -> io::Result<()> { + *self = Self::try_from(self.params.clone())?; + Ok(()) } - fn finish( + fn decode( &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> io::Result { - let previous_out = self.stream.total_out() as usize; + let previous_in = self.stream.total_in() as usize; - let status = self - .stream - .process(&[], output.unwritten_mut(), Action::Finish)?; + let res = self.process(input.unwritten(), output, Action::Run); + input.advance(self.stream.total_in() as usize - previous_in); - output.advance(self.stream.total_out() as usize - previous_out); + res + } - match status { - Status::Ok => Ok(false), - Status::StreamEnd => Ok(true), - Status::GetCheck => Err(io::Error::other("Unexpected lzma integrity check")), - Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()), - } + fn flush(&mut self, _output: &mut WriteBuffer<'_>) -> io::Result { + // While decoding flush is a noop + Ok(true) + } + + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { + self.process(&[], output, Action::Finish) } } diff --git a/crates/compression-codecs/src/xz2/encoder.rs b/crates/compression-codecs/src/xz2/encoder.rs index 4cdc4cc..84cc931 100644 --- a/crates/compression-codecs/src/xz2/encoder.rs +++ b/crates/compression-codecs/src/xz2/encoder.rs @@ -1,4 +1,7 @@ -use compression_core::{util::PartialBuffer, Level}; +use compression_core::{ + util::{PartialBuffer, WriteBuffer}, + Level, +}; use liblzma::stream::{Action, Check, Status, Stream}; use std::{ convert::{TryFrom, TryInto}, @@ -7,7 +10,7 @@ use std::{ use crate::{ lzma::params::{LzmaEncoderParams, LzmaOptions}, - Encode, Xz2FileFormat, + EncodeV2, Xz2FileFormat, }; /// Xz2 encoding stream @@ -74,37 +77,46 @@ impl Xz2Encoder { let params = LzmaEncoderParams::MultiThread { builder }; Self::try_from(params).unwrap() } -} -impl Encode for Xz2Encoder { - fn encode( + /// Return `Ok(true)` if stream ends. + fn process( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result<()> { + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, + action: Action, + ) -> io::Result { let previous_in = self.stream.total_in() as usize; let previous_out = self.stream.total_out() as usize; - let status = self - .stream - .process(input.unwritten(), output.unwritten_mut(), Action::Run)?; + output.initialize_unwritten(); + let res = self.stream.process( + input.unwritten(), + output.unwritten_initialized_mut(), + action, + ); input.advance(self.stream.total_in() as usize - previous_in); output.advance(self.stream.total_out() as usize - previous_out); - match status { - Status::Ok | Status::StreamEnd => Ok(()), + match res? { + Status::Ok => Ok(false), + Status::StreamEnd => Ok(true), Status::GetCheck => Err(io::Error::other("Unexpected lzma integrity check")), Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()), } } +} - fn flush( +impl EncodeV2 for Xz2Encoder { + fn encode( &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { - let previous_out = self.stream.total_out() as usize; + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, + ) -> io::Result<()> { + self.process(input, output, Action::Run).map(|_| ()) + } + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { let action = match &self.params { // Multi-threaded streams don't support SyncFlush, use FullFlush instead #[cfg(feature = "xz-parallel")] @@ -112,35 +124,10 @@ impl Encode for Xz2Encoder { _ => Action::SyncFlush, }; - let status = self.stream.process(&[], output.unwritten_mut(), action)?; - - output.advance(self.stream.total_out() as usize - previous_out); - - match status { - Status::Ok => Ok(false), - Status::StreamEnd => Ok(true), - Status::GetCheck => Err(io::Error::other("Unexpected lzma integrity check")), - Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()), - } + self.process(&mut PartialBuffer::new(&[]), output, action) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> io::Result { - let previous_out = self.stream.total_out() as usize; - - let status = self - .stream - .process(&[], output.unwritten_mut(), Action::Finish)?; - - output.advance(self.stream.total_out() as usize - previous_out); - - match status { - Status::Ok => Ok(false), - Status::StreamEnd => Ok(true), - Status::GetCheck => Err(io::Error::other("Unexpected lzma integrity check")), - Status::MemNeeded => Err(io::ErrorKind::OutOfMemory.into()), - } + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> io::Result { + self.process(&mut PartialBuffer::new(&[]), output, Action::Finish) } } diff --git a/crates/compression-codecs/src/zlib/decoder.rs b/crates/compression-codecs/src/zlib/decoder.rs index bd72fcc..7f0bf10 100644 --- a/crates/compression-codecs/src/zlib/decoder.rs +++ b/crates/compression-codecs/src/zlib/decoder.rs @@ -1,5 +1,5 @@ -use crate::{Decode, FlateDecoder}; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, FlateDecoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::io::Result; #[derive(Debug)] @@ -19,7 +19,7 @@ impl ZlibDecoder { } } -impl Decode for ZlibDecoder { +impl DecodeV2 for ZlibDecoder { fn reinit(&mut self) -> Result<()> { self.inner.reinit()?; Ok(()) @@ -27,23 +27,17 @@ impl Decode for ZlibDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { self.inner.decode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/zlib/encoder.rs b/crates/compression-codecs/src/zlib/encoder.rs index bf567ea..011faae 100644 --- a/crates/compression-codecs/src/zlib/encoder.rs +++ b/crates/compression-codecs/src/zlib/encoder.rs @@ -1,5 +1,5 @@ -use crate::{flate::params::FlateEncoderParams, Encode, FlateEncoder}; -use compression_core::util::PartialBuffer; +use crate::{flate::params::FlateEncoderParams, EncodeV2, FlateEncoder}; +use compression_core::util::{PartialBuffer, WriteBuffer}; use std::io::Result; #[derive(Debug)] @@ -19,26 +19,20 @@ impl ZlibEncoder { } } -impl Encode for ZlibEncoder { +impl EncodeV2 for ZlibEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { self.inner.encode(input, output) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.flush(output) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { self.inner.finish(output) } } diff --git a/crates/compression-codecs/src/zstd/decoder.rs b/crates/compression-codecs/src/zstd/decoder.rs index 2eea1aa..44e72ef 100644 --- a/crates/compression-codecs/src/zstd/decoder.rs +++ b/crates/compression-codecs/src/zstd/decoder.rs @@ -1,7 +1,9 @@ use crate::zstd::params::DParameter; -use crate::{Decode, DecodedSize}; -use compression_core::unshared::Unshared; -use compression_core::util::PartialBuffer; +use crate::{DecodeV2, DecodedSize}; +use compression_core::{ + unshared::Unshared, + util::{PartialBuffer, WriteBuffer}, +}; use libzstd::stream::raw::{Decoder, Operation}; use std::convert::TryInto; use std::io; @@ -42,9 +44,24 @@ impl ZstdDecoder { decoder: Unshared::new(decoder), }) } + + fn call_fn_on_out_buffer( + &mut self, + output: &mut WriteBuffer<'_>, + f: fn(&mut Decoder<'static>, &mut zstd_safe::OutBuffer<'_, [u8]>) -> io::Result, + ) -> io::Result { + output.initialize_unwritten(); + + let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_initialized_mut()); + let res = f(self.decoder.get_mut(), &mut out_buf); + let len = out_buf.as_slice().len(); + output.advance(len); + + res.map(|bytes_left| bytes_left == 0) + } } -impl Decode for ZstdDecoder { +impl DecodeV2 for ZstdDecoder { fn reinit(&mut self) -> Result<()> { self.decoder.get_mut().reinit()?; Ok(()) @@ -52,38 +69,26 @@ impl Decode for ZstdDecoder { fn decode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result { + output.initialize_unwritten(); + let status = self .decoder .get_mut() - .run_on_buffers(input.unwritten(), output.unwritten_mut())?; + .run_on_buffers(input.unwritten(), output.unwritten_initialized_mut())?; input.advance(status.bytes_read); output.advance(status.bytes_written); Ok(status.remaining == 0) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_mut()); - let bytes_left = self.decoder.get_mut().flush(&mut out_buf)?; - let len = out_buf.as_slice().len(); - output.advance(len); - Ok(bytes_left == 0) + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.call_fn_on_out_buffer(output, |decoder, out_buf| decoder.flush(out_buf)) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_mut()); - let bytes_left = self.decoder.get_mut().finish(&mut out_buf, true)?; - let len = out_buf.as_slice().len(); - output.advance(len); - Ok(bytes_left == 0) + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.call_fn_on_out_buffer(output, |decoder, out_buf| decoder.finish(out_buf, true)) } } diff --git a/crates/compression-codecs/src/zstd/encoder.rs b/crates/compression-codecs/src/zstd/encoder.rs index f7695d0..8737092 100644 --- a/crates/compression-codecs/src/zstd/encoder.rs +++ b/crates/compression-codecs/src/zstd/encoder.rs @@ -1,7 +1,9 @@ use crate::zstd::params::CParameter; -use crate::Encode; -use compression_core::unshared::Unshared; -use compression_core::util::PartialBuffer; +use crate::EncodeV2; +use compression_core::{ + unshared::Unshared, + util::{PartialBuffer, WriteBuffer}, +}; use libzstd::stream::raw::{Encoder, Operation}; use std::io; use std::io::Result; @@ -34,42 +36,45 @@ impl ZstdEncoder { encoder: Unshared::new(encoder), }) } + + fn call_fn_on_out_buffer( + &mut self, + output: &mut WriteBuffer<'_>, + f: fn(&mut Encoder<'static>, &mut zstd_safe::OutBuffer<'_, [u8]>) -> io::Result, + ) -> io::Result { + output.initialize_unwritten(); + + let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_initialized_mut()); + let res = f(self.encoder.get_mut(), &mut out_buf); + let len = out_buf.as_slice().len(); + output.advance(len); + + res.map(|bytes_left| bytes_left == 0) + } } -impl Encode for ZstdEncoder { +impl EncodeV2 for ZstdEncoder { fn encode( &mut self, - input: &mut PartialBuffer>, - output: &mut PartialBuffer + AsMut<[u8]>>, + input: &mut PartialBuffer<&[u8]>, + output: &mut WriteBuffer<'_>, ) -> Result<()> { + output.initialize_unwritten(); + let status = self .encoder .get_mut() - .run_on_buffers(input.unwritten(), output.unwritten_mut())?; + .run_on_buffers(input.unwritten(), output.unwritten_initialized_mut())?; input.advance(status.bytes_read); output.advance(status.bytes_written); Ok(()) } - fn flush( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_mut()); - let bytes_left = self.encoder.get_mut().flush(&mut out_buf)?; - let len = out_buf.as_slice().len(); - output.advance(len); - Ok(bytes_left == 0) + fn flush(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.call_fn_on_out_buffer(output, |encoder, out_buf| encoder.flush(out_buf)) } - fn finish( - &mut self, - output: &mut PartialBuffer + AsMut<[u8]>>, - ) -> Result { - let mut out_buf = zstd_safe::OutBuffer::around(output.unwritten_mut()); - let bytes_left = self.encoder.get_mut().finish(&mut out_buf, true)?; - let len = out_buf.as_slice().len(); - output.advance(len); - Ok(bytes_left == 0) + fn finish(&mut self, output: &mut WriteBuffer<'_>) -> Result { + self.call_fn_on_out_buffer(output, |encoder, out_buf| encoder.finish(out_buf, true)) } } diff --git a/crates/compression-core/src/util.rs b/crates/compression-core/src/util.rs index 58720a4..bf8b09c 100644 --- a/crates/compression-core/src/util.rs +++ b/crates/compression-core/src/util.rs @@ -16,12 +16,18 @@ impl> PartialBuffer { &self.buffer.as_ref()[..self.index] } + /// Convenient method for `.writen().len()` + pub fn written_len(&self) -> usize { + self.index + } + pub fn unwritten(&self) -> &[u8] { &self.buffer.as_ref()[self.index..] } pub fn advance(&mut self, amount: usize) { self.index += amount; + debug_assert!(self.index <= self.buffer.as_ref().len()); } pub fn get_mut(&mut self) -> &mut B { @@ -64,3 +70,73 @@ impl + AsMut<[u8]>> From for PartialBuffer { Self::new(buffer) } } + +/// Write buffer for compression-codecs. +/// +/// Currently it only supports initialized buffer, but will support uninitialized +/// buffer soon. +/// +/// # Layout +/// +/// ```text +/// | buffer | +/// | written and initialized | unwritten but initialized | unwritten and uninitialized +/// ``` +#[derive(Debug)] +pub struct WriteBuffer<'a> { + buffer: &'a mut [u8], + index: usize, +} + +impl<'a> WriteBuffer<'a> { + pub fn new_initialized(buffer: &'a mut [u8]) -> Self { + Self { buffer, index: 0 } + } + + pub fn written(&self) -> &[u8] { + &self.buffer[..self.index] + } + + /// Convenient method for `.writen().len()` + pub fn written_len(&self) -> usize { + self.index + } + + /// Buffer has no spare space to write any data + pub fn has_no_spare_space(&self) -> bool { + self.index == self.buffer.len() + } + + /// Initialize all uninitialized, unwritten part to initialized, unwritten part + pub fn initialize_unwritten(&mut self) {} + + /// Return initialized but unwritten part. + pub fn unwritten_initialized_mut(&mut self) -> &mut [u8] { + &mut self.buffer[self.index..] + } + + /// Advance written index within initialized part. + /// + /// Note that try to advance into uninitialized part would panic. + pub fn advance(&mut self, amount: usize) { + self.index += amount; + debug_assert!(self.index <= self.buffer.len()); + } + + pub fn reset(&mut self) { + self.index = 0; + } + + pub fn copy_unwritten_from>(&mut self, other: &mut PartialBuffer) -> usize { + let len = self + .unwritten_initialized_mut() + .len() + .min(other.unwritten().len()); + + self.unwritten_initialized_mut()[..len].copy_from_slice(&other.unwritten()[..len]); + + self.advance(len); + other.advance(len); + len + } +}