Separate codecs as a separate crate, allow direct configuration

* git mv all codecs into a new `compression-codecs` crate to retain git history
* Add missing constructors to all codecs to allow custom configuration
* Exposes custom codec constructions with `with_codec` as an alternative
* Updated `README.md` to include PR checks

Fixes #324, #359

Author: jkushmaul

.gitignore

@@ -1,6 +1,9 @@
 /target
 **/*.rs.bk
 
+wasi-sysroot*.tar.gz
+wasi-sysroot/
+
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock

Cargo.toml

@@ -1,16 +1,32 @@
-[package]
-name = "async-compression"
-version = "0.4.27"
-authors = ["Wim Looman <[email protected]>", "Allen Bui <[email protected]>"]
-edition = "2018"
+[workspace]
+members = ["crates/*"]
 resolver = "2"
+
+[workspace.package]
+authors = ["Wim Looman <[email protected]>", "Allen Bui <[email protected]>"]
 license = "MIT OR Apache-2.0"
-keywords = ["compression", "gzip", "zstd", "brotli", "async"]
-categories = ["compression", "asynchronous"]
 repository = "https://github.com/Nullus157/async-compression"
+categories = ["compression", "asynchronous"]
+version = "0.4.27"
+edition = "2018"
+
+[workspace.dependencies]
+compression-codecs = { path = "crates/compression-codecs" }
+compression-core = { path = "crates/compression-core" }
+futures-core = { version = "0.3", default-features = false }
+memchr = "2"
+pin-project-lite = "0.2"
+
+[package]
+name = "async-compression"
 description = """
 Adaptors between compression crates and Rust's modern asynchronous IO types.
 """
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+categories.workspace = true
+edition.workspace = true
 
 [package.metadata.docs.rs]
 all-features = true
@@ -20,31 +36,52 @@ rustdoc-args = ["--cfg", "docsrs"]
 # groups
 all = ["all-implementations", "all-algorithms"]
 all-implementations = ["futures-io", "tokio"]
-all-algorithms = ["brotli", "bzip2", "deflate", "gzip", "lz4", "lzma", "xz-parallel", "xz", "zlib", "zstd", "deflate64"]
+all-algorithms = [
+    "brotli",
+    "bzip2",
+    "deflate",
+    "deflate64",
+    "gzip",
+    "lz4",
+    "lzma",
+    "xz",
+    "xz-parallel",
+    "zlib",
+    "zstd",
+]
 
 # algorithms
-deflate = ["flate2"]
-gzip = ["flate2"]
-lz4 = ["dep:lz4"]
-lzma = ["dep:liblzma"]
-xz = ["lzma"]
-xz-parallel = ["xz", "liblzma/parallel"]
-xz2 = ["xz"]
-zlib = ["flate2"]
-zstd = ["libzstd", "zstd-safe"]
-zstdmt = ["zstd", "zstd-safe/zstdmt"]
-deflate64 = ["dep:deflate64"]
+brotli = ["compression-codecs/brotli", "dep:brotli"]
+bzip2 = ["compression-codecs/bzip2", "dep:bzip2"]
+deflate = ["compression-codecs/deflate", "flate2"]
+deflate64 = ["compression-codecs/deflate64", "dep:deflate64"]
+gzip = ["compression-codecs/gzip", "flate2"]
+lz4 = ["compression-codecs/lz4", "dep:lz4"]
+lzma = ["compression-codecs/lzma", "liblzma"]
+xz = ["compression-codecs/xz", "lzma"]
+xz-parallel = ["compression-codecs/xz-parallel", "xz", "liblzma/parallel"]
+xz2 = ["compression-codecs/xz2", "xz"]
+zlib = ["compression-codecs/zlib", "flate2"]
+zstd = ["compression-codecs/zstd", "libzstd", "zstd-safe"]
+zstdmt = ["compression-codecs/zstdmt", "zstd", "zstd-safe/zstdmt"]
+
 
 [dependencies]
+# core dependencies
+futures-core.workspace = true
+memchr.workspace = true
+pin-project-lite.workspace = true
+compression-codecs.workspace = true
+compression-core.workspace = true
+# optionals deps
 brotli = { version = "8", optional = true }
 bzip2 = { version = "0.6", optional = true }
 flate2 = { version = "1.0.13", optional = true }
-futures-core = { version = "0.3", default-features = false }
-futures-io = { version = "0.3", default-features = false, features = ["std"], optional = true }
+futures-io = { version = "0.3", default-features = false, features = [
+    "std",
+], optional = true }
 libzstd = { package = "zstd", version = "0.13.1", optional = true, default-features = false }
 lz4 = { version = "1.28.1", optional = true }
-memchr = "2"
-pin-project-lite = "0.2"
 tokio = { version = "1.24.2", optional = true, default-features = false }
 liblzma = { version = "0.4.2", optional = true }
 zstd-safe = { version = "7", optional = true, default-features = false }
@@ -58,7 +95,12 @@ ntest = "0.9"
 proptest = "1"
 proptest-derive = "0.6"
 rand = "0.9"
-tokio = { version = "1.38.2", default-features = false, features = ["io-util", "macros", "rt-multi-thread", "io-std"] }
+tokio = { version = "1.38.2", default-features = false, features = [
+    "io-util",
+    "macros",
+    "rt-multi-thread",
+    "io-std",
+] }
 tokio-util = { version = "0.7", default-features = false, features = ["io"] }
 
 [[test]]
@@ -112,3 +154,8 @@ required-features = ["zlib", "tokio"]
 [[example]]
 name = "zstd_gzip"
 required-features = ["zstd", "gzip", "tokio"]
+
+
+[[example]]
+name = "lzma_filters"
+required-features = ["xz", "tokio"]

README.md

@@ -20,6 +20,43 @@ enable different subsets of features as appropriate for the code you are
 testing to avoid compiling all dependencies, e.g. `cargo test --features
 tokio,gzip`.
 
+To prepare for a pull request, you can run several other checks:
+
+1. `fmt`
+
+    ```bash
+    cargo fmt --all
+    cargo clippy --no-deps
+    ```
+
+2. `build`
+
+    ```bash
+    cargo build --lib --all-features
+    ```
+
+3. `nextest`
+
+    ```bash
+    cargo --locked nextest run --workspace --all-features
+    ```
+
+4. `hack check`
+
+    ```bash
+    cargo hack check --workspace --feature-powerset --all-targets --skip 'all,all-algorithms,all-implementations'
+    ```
+
+5. `wasm32` - Linux only
+
+    ```bash
+    gh release download --repo WebAssembly/wasi-sdk --pattern 'wasi-sysroot-*.tar.gz'
+    rustup target add wasm32-wasip1-threads
+
+    export "CFLAGS_wasm32_wasip1_threads=--sysroot=\"${PWD}/wasi-sysroot\" -I\"${PWD}/wasi-sysroot/include/wasm32-wasip1-threads\" -L-I\"${PWD}/wasi-sysroot/lib/wasm32-wasip1-threads\""
+    cargo build --lib --features all-implementations,brotli,bzip2,deflate,gzip,lz4,lzma,xz,zlib,zstd,deflate64 --target wasm32-wasip1-threads
+    ```
+
 ## License
 
 Licensed under either of

crates/compression-codecs/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "compression-codecs"
+description = """
+Adaptors for various compression algorithms.
+"""
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+categories.workspace = true
+edition.workspace = true
+
+[features]
+all-algorithms = [
+    "brotli",
+    "bzip2",
+    "deflate",
+    "gzip",
+    "lz4",
+    "lzma",
+    "xz-parallel",
+    "xz",
+    "zlib",
+    "zstd",
+    "deflate64",
+]
+
+# algorithms
+deflate = ["flate2"]
+gzip = ["flate2"]
+lz4 = ["dep:lz4"]
+lzma = ["dep:liblzma"]
+xz = ["lzma"]
+xz-parallel = ["xz", "liblzma/parallel"]
+xz2 = ["xz"]
+zlib = ["flate2"]
+zstd = ["libzstd", "zstd-safe"]
+zstdmt = ["zstd", "zstd-safe/zstdmt"]
+deflate64 = ["dep:deflate64"]
+
+
+[dependencies]
+# Workspace dependencies.
+compression-core.workspace = true
+futures-core.workspace = true
+memchr.workspace = true
+pin-project-lite.workspace = true
+# features
+brotli = { version = "8", optional = true }
+bzip2 = { version = "0.6", optional = true }
+flate2 = { version = "1.0.13", optional = true }
+libzstd = { package = "zstd", version = "0.13.1", optional = true, default-features = false }
+lz4 = { version = "1.28.1", optional = true }
+liblzma = { version = "0.4.2", optional = true }
+zstd-safe = { version = "7", optional = true, default-features = false }
+deflate64 = { version = "0.1.5", optional = true }

src/codec/brotli/decoder.rs renamed to crates/compression-codecs/src/brotli/decoder.rs

@@ -1,15 +1,15 @@
-use crate::{codec::Decode, util::PartialBuffer};
-use std::{fmt, io};
-
+use crate::Decode;
 use brotli::{enc::StandardAlloc, BrotliDecompressStream, BrotliResult, BrotliState};
+use compression_core::util::PartialBuffer;
+use std::{fmt, io};
 
 pub struct BrotliDecoder {
     // `BrotliState` is very large (over 2kb) which is why we're boxing it.
     state: Box<BrotliState<StandardAlloc, StandardAlloc, StandardAlloc>>,
 }
 
-impl BrotliDecoder {
-    pub(crate) fn new() -> Self {
+impl Default for BrotliDecoder {
+    fn default() -> Self {
         Self {
             state: Box::new(BrotliState::new(
                 StandardAlloc::default(),
@@ -18,14 +18,20 @@ impl BrotliDecoder {
             )),
         }
     }
+}
+
+impl BrotliDecoder {
+    pub fn new() -> Self {
+        Self::default()
+    }
 
     fn decode(
         &mut self,
         input: &mut PartialBuffer<impl AsRef<[u8]>>,
         output: &mut PartialBuffer<impl AsRef<[u8]> + AsMut<[u8]>>,
     ) -> io::Result<BrotliResult> {
         let in_buf = input.unwritten();
-        let mut out_buf = output.unwritten_mut();
+        let out_buf = output.unwritten_mut();
 
         let mut input_len = 0;
         let mut output_len = 0;

src/codec/brotli/encoder.rs renamed to crates/compression-codecs/src/brotli/encoder.rs

@@ -1,18 +1,19 @@
-use crate::{codec::Encode, util::PartialBuffer};
-use std::{fmt, io};
-
+use crate::{brotli::params::EncoderParams, Encode};
 use brotli::enc::{
     backward_references::BrotliEncoderParams,
     encode::{BrotliEncoderOperation, BrotliEncoderStateStruct},
     StandardAlloc,
 };
+use compression_core::util::PartialBuffer;
+use std::{fmt, io};
 
 pub struct BrotliEncoder {
     state: BrotliEncoderStateStruct<StandardAlloc>,
 }
 
 impl BrotliEncoder {
-    pub(crate) fn new(params: BrotliEncoderParams) -> Self {
+    pub fn new(params: EncoderParams) -> Self {
+        let params = BrotliEncoderParams::from(params);
         let mut state = BrotliEncoderStateStruct::new(StandardAlloc::default());
         state.params = params;
         Self { state }
@@ -25,7 +26,7 @@ impl BrotliEncoder {
         op: BrotliEncoderOperation,
     ) -> io::Result<()> {
         let in_buf = input.unwritten();
-        let mut out_buf = output.unwritten_mut();
+        let out_buf = output.unwritten_mut();
 
         let mut input_len = 0;
         let mut output_len = 0;

crates/compression-codecs/src/brotli/mod.rs

@@ -0,0 +1,5 @@
+mod decoder;
+mod encoder;
+pub mod params;
+
+pub use self::{decoder::BrotliDecoder, encoder::BrotliEncoder};