-
Notifications
You must be signed in to change notification settings - Fork 182
add zlib-rs support via the libz-rs-sys C api for zlib-rs
#400
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ and raw deflate streams. | |
| [dependencies] | ||
| libz-sys = { version = "1.1.8", optional = true, default-features = false } | ||
| libz-ng-sys = { version = "1.1.8", optional = true } | ||
| libz-rs-sys = { version = "0.1.1", optional = true, default-features = false, features = ["std", "rust-allocator"] } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes exactly. it also clarifies that we're explicitly using the rust allocator; that will always be the default when the crate is used from rust, but not when we build a C dynamic library. |
||
| cloudflare-zlib-sys = { version = "0.3.0", optional = true } | ||
| miniz_oxide = { version = "0.7.1", optional = true, default-features = false, features = ["with-alloc"] } | ||
| crc32fast = "1.2.0" | ||
|
|
@@ -38,6 +39,7 @@ zlib = ["any_zlib", "libz-sys"] | |
| zlib-default = ["any_zlib", "libz-sys/default"] | ||
| zlib-ng-compat = ["zlib", "libz-sys/zlib-ng"] | ||
| zlib-ng = ["any_zlib", "libz-ng-sys"] | ||
| zlib-rs = ["any_zlib", "libz-rs-sys"] | ||
| cloudflare_zlib = ["any_zlib", "cloudflare-zlib-sys"] | ||
| rust_backend = ["miniz_oxide", "any_impl"] | ||
| miniz-sys = ["rust_backend"] # For backwards compatibility | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,8 @@ | ||
| //! Implementation for C backends. | ||
| use std::alloc::{self, Layout}; | ||
| use std::cmp; | ||
| use std::convert::TryFrom; | ||
| use std::fmt; | ||
| use std::marker; | ||
| use std::os::raw::{c_int, c_uint, c_void}; | ||
| use std::os::raw::{c_int, c_uint}; | ||
| use std::ptr; | ||
|
|
||
| use super::*; | ||
|
|
@@ -52,14 +50,28 @@ impl Default for StreamWrapper { | |
| reserved: 0, | ||
| opaque: ptr::null_mut(), | ||
| state: ptr::null_mut(), | ||
| #[cfg(all(feature = "any_zlib", not(feature = "cloudflare-zlib-sys")))] | ||
| zalloc, | ||
| #[cfg(all(feature = "any_zlib", not(feature = "cloudflare-zlib-sys")))] | ||
| zfree, | ||
| #[cfg(not(all(feature = "any_zlib", not(feature = "cloudflare-zlib-sys"))))] | ||
| zalloc: Some(zalloc), | ||
| #[cfg(not(all(feature = "any_zlib", not(feature = "cloudflare-zlib-sys"))))] | ||
| zfree: Some(zfree), | ||
| #[cfg(all( | ||
| feature = "any_zlib", | ||
| not(any(feature = "cloudflare-zlib-sys", feature = "libz-rs-sys")) | ||
| ))] | ||
| zalloc: allocator::zalloc, | ||
| #[cfg(all( | ||
| feature = "any_zlib", | ||
| not(any(feature = "cloudflare-zlib-sys", feature = "libz-rs-sys")) | ||
| ))] | ||
| zfree: allocator::zfree, | ||
|
|
||
| #[cfg(all(feature = "any_zlib", feature = "cloudflare-zlib-sys"))] | ||
| zalloc: Some(allocator::zalloc), | ||
| #[cfg(all(feature = "any_zlib", feature = "cloudflare-zlib-sys"))] | ||
| zfree: Some(allocator::zfree), | ||
|
|
||
| // for zlib-rs, it is most efficient to have it provide the allocator. | ||
| // The libz-rs-sys dependency is configured to use the rust system allocator | ||
| #[cfg(all(feature = "any_zlib", feature = "libz-rs-sys"))] | ||
| zalloc: None, | ||
| #[cfg(all(feature = "any_zlib", feature = "libz-rs-sys"))] | ||
| zfree: None, | ||
|
Comment on lines
+69
to
+74
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the trick is that because zlib-rs can do equality comparisons on the function pointer, it can know when its built-in allocator is used, and this allocator guarantees 64-byte alignment. The allocation of |
||
| })), | ||
| } | ||
| } | ||
|
|
@@ -75,54 +87,63 @@ impl Drop for StreamWrapper { | |
| } | ||
| } | ||
|
|
||
| const ALIGN: usize = std::mem::align_of::<usize>(); | ||
| #[cfg(all(feature = "any_zlib", not(feature = "libz-rs-sys")))] | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. in the case of zlib-rs, the allocator logic is unused. That is an error for this crate, so it's only conditionally compiled now. |
||
| mod allocator { | ||
| use super::*; | ||
|
|
||
| fn align_up(size: usize, align: usize) -> usize { | ||
| (size + align - 1) & !(align - 1) | ||
| } | ||
| use std::alloc::{self, Layout}; | ||
| use std::convert::TryFrom; | ||
| use std::os::raw::c_void; | ||
|
|
||
| const ALIGN: usize = std::mem::align_of::<usize>(); | ||
|
|
||
| fn align_up(size: usize, align: usize) -> usize { | ||
| (size + align - 1) & !(align - 1) | ||
| } | ||
|
|
||
| pub extern "C" fn zalloc(_ptr: *mut c_void, items: uInt, item_size: uInt) -> *mut c_void { | ||
Byron marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // We need to multiply `items` and `item_size` to get the actual desired | ||
| // allocation size. Since `zfree` doesn't receive a size argument we | ||
| // also need to allocate space for a `usize` as a header so we can store | ||
| // how large the allocation is to deallocate later. | ||
| let size = match items | ||
| .checked_mul(item_size) | ||
| .and_then(|i| usize::try_from(i).ok()) | ||
| .map(|size| align_up(size, ALIGN)) | ||
| .and_then(|i| i.checked_add(std::mem::size_of::<usize>())) | ||
| { | ||
| Some(i) => i, | ||
| None => return ptr::null_mut(), | ||
| }; | ||
|
|
||
| // Make sure the `size` isn't too big to fail `Layout`'s restrictions | ||
| let layout = match Layout::from_size_align(size, ALIGN) { | ||
| Ok(layout) => layout, | ||
| Err(_) => return ptr::null_mut(), | ||
| }; | ||
|
|
||
| extern "C" fn zalloc(_ptr: *mut c_void, items: AllocSize, item_size: AllocSize) -> *mut c_void { | ||
| // We need to multiply `items` and `item_size` to get the actual desired | ||
| // allocation size. Since `zfree` doesn't receive a size argument we | ||
| // also need to allocate space for a `usize` as a header so we can store | ||
| // how large the allocation is to deallocate later. | ||
| let size = match items | ||
| .checked_mul(item_size) | ||
| .and_then(|i| usize::try_from(i).ok()) | ||
| .map(|size| align_up(size, ALIGN)) | ||
| .and_then(|i| i.checked_add(std::mem::size_of::<usize>())) | ||
| { | ||
| Some(i) => i, | ||
| None => return ptr::null_mut(), | ||
| }; | ||
|
|
||
| // Make sure the `size` isn't too big to fail `Layout`'s restrictions | ||
| let layout = match Layout::from_size_align(size, ALIGN) { | ||
| Ok(layout) => layout, | ||
| Err(_) => return ptr::null_mut(), | ||
| }; | ||
|
|
||
| unsafe { | ||
| // Allocate the data, and if successful store the size we allocated | ||
| // at the beginning and then return an offset pointer. | ||
| let ptr = alloc::alloc(layout) as *mut usize; | ||
| if ptr.is_null() { | ||
| return ptr as *mut c_void; | ||
| unsafe { | ||
| // Allocate the data, and if successful store the size we allocated | ||
| // at the beginning and then return an offset pointer. | ||
| let ptr = alloc::alloc(layout) as *mut usize; | ||
| if ptr.is_null() { | ||
| return ptr as *mut c_void; | ||
| } | ||
| *ptr = size; | ||
| ptr.add(1) as *mut c_void | ||
| } | ||
| *ptr = size; | ||
| ptr.add(1) as *mut c_void | ||
| } | ||
| } | ||
|
|
||
| extern "C" fn zfree(_ptr: *mut c_void, address: *mut c_void) { | ||
| unsafe { | ||
| // Move our address being freed back one pointer, read the size we | ||
| // stored in `zalloc`, and then free it using the standard Rust | ||
| // allocator. | ||
| let ptr = (address as *mut usize).offset(-1); | ||
| let size = *ptr; | ||
| let layout = Layout::from_size_align_unchecked(size, ALIGN); | ||
| alloc::dealloc(ptr as *mut u8, layout) | ||
| pub extern "C" fn zfree(_ptr: *mut c_void, address: *mut c_void) { | ||
| unsafe { | ||
| // Move our address being freed back one pointer, read the size we | ||
| // stored in `zalloc`, and then free it using the standard Rust | ||
| // allocator. | ||
| let ptr = (address as *mut usize).offset(-1); | ||
| let size = *ptr; | ||
| let layout = Layout::from_size_align_unchecked(size, ALIGN); | ||
| alloc::dealloc(ptr as *mut u8, layout) | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -382,10 +403,17 @@ mod c_backend { | |
| #[cfg(feature = "zlib-ng")] | ||
| use libz_ng_sys as libz; | ||
|
|
||
| #[cfg(feature = "zlib-rs")] | ||
| use libz_rs_sys as libz; | ||
|
|
||
| #[cfg(all(not(feature = "zlib-ng"), feature = "cloudflare_zlib"))] | ||
| use cloudflare_zlib_sys as libz; | ||
|
|
||
| #[cfg(all(not(feature = "cloudflare_zlib"), not(feature = "zlib-ng")))] | ||
| #[cfg(all( | ||
| not(feature = "cloudflare_zlib"), | ||
| not(feature = "zlib-ng"), | ||
| not(feature = "zlib-rs") | ||
| ))] | ||
| use libz_sys as libz; | ||
|
|
||
| pub use libz::deflate as mz_deflate; | ||
|
|
@@ -410,13 +438,14 @@ mod c_backend { | |
| pub use libz::Z_STREAM_END as MZ_STREAM_END; | ||
| pub use libz::Z_STREAM_ERROR as MZ_STREAM_ERROR; | ||
| pub use libz::Z_SYNC_FLUSH as MZ_SYNC_FLUSH; | ||
| pub type AllocSize = libz::uInt; | ||
|
|
||
| pub const MZ_DEFAULT_WINDOW_BITS: c_int = 15; | ||
|
|
||
| #[cfg(feature = "zlib-ng")] | ||
| const ZLIB_VERSION: &'static str = "2.1.0.devel\0"; | ||
| #[cfg(not(feature = "zlib-ng"))] | ||
| #[cfg(feature = "zlib-rs")] | ||
| const ZLIB_VERSION: &'static str = "0.1.0\0"; | ||
Byron marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| #[cfg(not(any(feature = "zlib-ng", feature = "zlib-rs")))] | ||
| const ZLIB_VERSION: &'static str = "1.2.8\0"; | ||
|
|
||
| pub unsafe extern "C" fn mz_deflateInit2( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
we should eventually be able to compile on mingw, but the current release doesn't, and we don't test for it, so disabling for now.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This will need fixing before we could consider it as a potential default, but it's fine for now.