diff --git a/.travis.yml b/.travis.yml index c41f4b36d..04c90605b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,3 +24,4 @@ script: - cd uint/ && cargo test --features=std,quickcheck --release && cd .. - cd plain_hasher/ && cargo test --no-default-features && cd .. - cd parity-util-mem/ && cargo test --features=estimate-heapsize && cd .. + - cd parity-util-mem/ && cargo test --features=jemalloc-global && cd .. diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f34677aa2..6293d6348 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -12,15 +12,18 @@ cfg-if = "0.1.6" malloc_size_of_derive = "0.1.0" dlmalloc = { version = "0.1", features = ["global"], optional = true } wee_alloc = { version = "0.4", optional = true } -jemallocator = { version = "0.1", optional = true } elastic-array = { version = "*", optional = true } ethereum-types = { version = "*", optional = true } parking_lot = { version = "*", optional = true } -[target."cfg(windows)".dependencies.winapi] +[target.'cfg(target_os = "windows")'.dependencies.winapi] version = "0.3.4" +[target.'cfg(not(target_os = "windows"))'.dependencies.jemallocator] +version = "0.1" +optional = true + [features] default = ["std", "ethereum-impls"] std = [] diff --git a/parity-util-mem/slim_malloc_size_of.patch b/parity-util-mem/slim_malloc_size_of.patch index 61381c146..7c31cf606 100644 --- a/parity-util-mem/slim_malloc_size_of.patch +++ b/parity-util-mem/slim_malloc_size_of.patch @@ -1,5 +1,5 @@ diff --git a/components/malloc_size_of/lib.rs b/components/malloc_size_of/lib.rs -index 778082b5f0..7f527c930e 100644 +index 778082b5f0..e13745d6af 100644 --- a/components/malloc_size_of/lib.rs +++ b/components/malloc_size_of/lib.rs @@ -43,55 +43,39 @@ @@ -82,23 +82,67 @@ index 778082b5f0..7f527c930e 100644 /// Operations used when measuring heap usage of data structures. pub struct MallocSizeOfOps { -@@ -216,6 +200,7 @@ pub trait MallocConditionalShallowSizeOf { +@@ -216,44 +200,62 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocSizeOf for String { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -229,6 +214,7 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { +-impl MallocSizeOf for String { +- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.as_ptr()) } ++#[cfg(not(any( ++ all( ++ target_os = "macos", ++ not(feature = "jemalloc-global"), ++ ), ++ feature = "estimate-heapsize" ++)))] ++pub mod inner_allocator_use { ++ ++use super::*; ++ ++impl MallocShallowSizeOf for Box { ++ fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(&**self) } + } + } + +-impl<'a, T: ?Sized> MallocSizeOf for &'a T { +- fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { +- // Zero makes sense for a non-owning reference. +- 0 ++impl MallocShallowSizeOf for Vec { ++ fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(self.as_ptr()) } + } + } + +-impl MallocShallowSizeOf for Box { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(&**self) } ++// currently this seems only fine with jemalloc ++#[cfg(feature = "std")] ++#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] ++impl MallocUnconditionalShallowSizeOf for Arc { ++ fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { ++ unsafe { ops.malloc_size_of(arc_ptr(self)) } } } -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocShallowSizeOf for Box { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(&**self) } -@@ -241,24 +227,6 @@ impl MallocSizeOf for Box { +-impl MallocSizeOf for Box { ++#[cfg(feature = "std")] ++#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] ++impl MallocUnconditionalShallowSizeOf for Arc { ++ fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { ++ size_of::() ++ } ++} ++ ++} ++ ++impl MallocSizeOf for String { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- self.shallow_size_of(ops) + (**self).size_of(ops) ++ unsafe { ops.malloc_size_of(self.as_ptr()) } } } @@ -111,19 +155,19 @@ index 778082b5f0..7f527c930e 100644 - n += ops.malloc_size_of(&**self); - } - n -- } --} -- ++impl<'a, T: ?Sized> MallocSizeOf for &'a T { ++ fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { ++ // Zero makes sense for a non-owning reference. ++ 0 + } + } + -impl MallocSizeOf for thin_slice::ThinBoxedSlice { -- fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { -- self.shallow_size_of(ops) + (**self).size_of(ops) -- } --} -- - impl MallocSizeOf for () { - fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - 0 -@@ -329,6 +297,7 @@ impl MallocSizeOf for std::cell::RefCell { ++impl MallocSizeOf for Box { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + self.shallow_size_of(ops) + (**self).size_of(ops) + } +@@ -329,6 +331,7 @@ impl MallocSizeOf for std::cell::RefCell { } } @@ -131,7 +175,7 @@ index 778082b5f0..7f527c930e 100644 impl<'a, B: ?Sized + ToOwned> MallocSizeOf for std::borrow::Cow<'a, B> where B::Owned: MallocSizeOf, -@@ -351,24 +320,7 @@ impl MallocSizeOf for [T] { +@@ -351,30 +354,6 @@ impl MallocSizeOf for [T] { } } @@ -153,11 +197,16 @@ index 778082b5f0..7f527c930e 100644 - } -} - -+#[cfg(not(feature = "estimate-heapsize"))] - impl MallocShallowSizeOf for Vec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } -@@ -412,30 +364,7 @@ impl MallocSizeOf for std::collections::VecDeque { +-impl MallocShallowSizeOf for Vec { +- fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.as_ptr()) } +- } +-} +- + impl MallocSizeOf for Vec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); +@@ -412,30 +391,7 @@ impl MallocSizeOf for std::collections::VecDeque { } } @@ -189,7 +238,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocShallowSizeOf for std::collections::HashSet where T: Eq + Hash, -@@ -457,6 +386,7 @@ where +@@ -457,6 +413,7 @@ where } } @@ -197,7 +246,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocSizeOf for std::collections::HashSet where T: Eq + Hash + MallocSizeOf, -@@ -471,59 +401,7 @@ where +@@ -471,59 +428,7 @@ where } } @@ -258,7 +307,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocShallowSizeOf for std::collections::HashMap where K: Eq + Hash, -@@ -541,6 +419,7 @@ where +@@ -541,6 +446,7 @@ where } } @@ -266,7 +315,7 @@ index 778082b5f0..7f527c930e 100644 impl MallocSizeOf for std::collections::HashMap where K: Eq + Hash + MallocSizeOf, -@@ -587,62 +466,6 @@ where +@@ -587,62 +493,6 @@ where } } @@ -329,40 +378,21 @@ index 778082b5f0..7f527c930e 100644 // PhantomData is always 0. impl MallocSizeOf for std::marker::PhantomData { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -@@ -657,21 +480,43 @@ impl MallocSizeOf for std::marker::PhantomData { +@@ -657,21 +507,22 @@ impl MallocSizeOf for std::marker::PhantomData { //impl !MallocSizeOf for Arc { } //impl !MallocShallowSizeOf for Arc { } -impl MallocUnconditionalShallowSizeOf for servo_arc::Arc { +- fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { +- unsafe { ops.malloc_size_of(self.heap_ptr()) } +- } +#[cfg(feature = "std")] +fn arc_ptr(s: &Arc) -> * const T { + &(**s) as *const T -+} -+ -+ -+// currently this seems only fine with jemalloc -+#[cfg(feature = "std")] -+#[cfg(not(feature = "estimate-heapsize"))] -+#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] -+impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { -- unsafe { ops.malloc_size_of(self.heap_ptr()) } -+ unsafe { ops.malloc_size_of(arc_ptr(self)) } - } } -impl MallocUnconditionalSizeOf for servo_arc::Arc { +#[cfg(feature = "std")] -+#[cfg(not(feature = "estimate-heapsize"))] -+#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -+impl MallocUnconditionalShallowSizeOf for Arc { -+ fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { -+ size_of::() -+ } -+} -+ -+ -+#[cfg(feature = "std")] +impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.unconditional_shallow_size_of(ops) + (**self).size_of(ops) @@ -378,7 +408,7 @@ index 778082b5f0..7f527c930e 100644 0 } else { self.unconditional_shallow_size_of(ops) -@@ -679,9 +524,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { +@@ -679,9 +530,10 @@ impl MallocConditionalShallowSizeOf for servo_arc::Arc { } } @@ -391,7 +421,7 @@ index 778082b5f0..7f527c930e 100644 0 } else { self.unconditional_size_of(ops) -@@ -695,203 +541,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { +@@ -695,203 +547,13 @@ impl MallocConditionalSizeOf for servo_arc::Arc { /// If a mutex is stored inside of an Arc value as a member of a data type that is being measured, /// the Arc will not be automatically measured so there is no risk of overcounting the mutex's /// contents. @@ -596,7 +626,7 @@ index 778082b5f0..7f527c930e 100644 #[macro_export] macro_rules! malloc_size_of_is_0( ($($ty:ty),+) => ( -@@ -929,117 +585,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range +@@ -929,117 +591,6 @@ malloc_size_of_is_0!(Range, Range, Range, Range, Range malloc_size_of_is_0!(Range, Range, Range, Range, Range); malloc_size_of_is_0!(Range, Range); diff --git a/parity-util-mem/src/allocators.rs b/parity-util-mem/src/allocators.rs index ca8cea7b3..6f63b89fb 100644 --- a/parity-util-mem/src/allocators.rs +++ b/parity-util-mem/src/allocators.rs @@ -26,6 +26,11 @@ //! - jemalloc: use jemallocator crate //! - weealloc: default to `estimate_size` //! - dlmalloc: default to `estimate_size` +//! - arch x86/macos: +//! - no features: use default alloc, requires using `estimate_size` +//! - jemalloc: use jemallocator crate +//! - weealloc: default to `estimate_size` +//! - dlmalloc: default to `estimate_size` //! - arch wasm32: //! - no features: default to `estimate_size` //! - weealloc: default to `estimate_size` @@ -92,16 +97,19 @@ cfg_if! { jemallocator::usable_size(ptr) } - } else { + } else if #[cfg(target_os = "linux")] { - // default allocator used - /// Macos, ios and android calls jemalloc. /// Linux call system allocator (currently malloc). extern "C" { - #[cfg_attr(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android"), link_name = "je_malloc_usable_size")] pub fn malloc_usable_size(ptr: *const c_void) -> usize; } + } else { + // default allocator for non linux or windows system use estimate + pub unsafe extern "C" fn malloc_usable_size(_ptr: *const c_void) -> usize { + unreachable!("estimate heapsize or feature allocator needed") + } + } } diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index eafba2357..a07ff5c57 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -22,6 +22,7 @@ #![cfg_attr(not(feature = "std"), feature(core_intrinsics))] #![cfg_attr(not(feature = "std"), feature(alloc))] + #[macro_use] extern crate cfg_if; @@ -43,7 +44,6 @@ pub use cod::clear::Clear; cfg_if! { if #[cfg(all( - feature = "jemalloc-global", feature = "jemalloc-global", not(target_os = "windows"), not(target_arch = "wasm32") @@ -69,7 +69,13 @@ cfg_if! { pub mod allocators; -#[cfg(feature = "estimate-heapsize")] +#[cfg(any( + all( + target_os = "macos", + not(feature = "jemalloc-global"), + ), + feature = "estimate-heapsize" +))] pub mod sizeof; #[cfg(not(feature = "std"))] @@ -141,7 +147,7 @@ impl> DerefMut for Memzero { } } -#[cfg(std)] +#[cfg(feature = "std")] #[cfg(test)] mod test { use std::sync::Arc; @@ -153,5 +159,4 @@ mod test { let s = val.malloc_size_of(); assert!(s > 0); } - } diff --git a/parity-util-mem/src/malloc_size.rs b/parity-util-mem/src/malloc_size.rs index 7f527c930..ab9b0327b 100644 --- a/parity-util-mem/src/malloc_size.rs +++ b/parity-util-mem/src/malloc_size.rs @@ -200,13 +200,54 @@ pub trait MallocConditionalShallowSizeOf { fn conditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize; } -#[cfg(not(feature = "estimate-heapsize"))] +#[cfg(not(any( + all( + target_os = "macos", + not(feature = "jemalloc-global"), + ), + feature = "estimate-heapsize" +)))] +pub mod inner_allocator_use { + +use super::*; + +impl MallocShallowSizeOf for Box { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(&**self) } + } +} + +impl MallocShallowSizeOf for Vec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } +} + +// currently this seems only fine with jemalloc +#[cfg(feature = "std")] +#[cfg(all(feature = "jemalloc-global", not(target_os = "windows")))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + unsafe { ops.malloc_size_of(arc_ptr(self)) } + } +} + +#[cfg(feature = "std")] +#[cfg(not(all(feature = "jemalloc-global", not(target_os = "windows"))))] +impl MallocUnconditionalShallowSizeOf for Arc { + fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { + size_of::() + } +} + impl MallocSizeOf for String { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { unsafe { ops.malloc_size_of(self.as_ptr()) } } } +} + impl<'a, T: ?Sized> MallocSizeOf for &'a T { fn size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { // Zero makes sense for a non-owning reference. @@ -214,13 +255,6 @@ impl<'a, T: ?Sized> MallocSizeOf for &'a T { } } -#[cfg(not(feature = "estimate-heapsize"))] -impl MallocShallowSizeOf for Box { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(&**self) } - } -} - impl MallocSizeOf for Box { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { self.shallow_size_of(ops) + (**self).size_of(ops) @@ -320,13 +354,6 @@ impl MallocSizeOf for [T] { } } -#[cfg(not(feature = "estimate-heapsize"))] -impl MallocShallowSizeOf for Vec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } -} - impl MallocSizeOf for Vec { fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { let mut n = self.shallow_size_of(ops); @@ -485,27 +512,6 @@ fn arc_ptr(s: &Arc) -> * const T { &(**s) as *const T } - -// currently this seems only fine with jemalloc -#[cfg(feature = "std")] -#[cfg(not(feature = "estimate-heapsize"))] -#[cfg(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global"))] -impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - unsafe { ops.malloc_size_of(arc_ptr(self)) } - } -} - -#[cfg(feature = "std")] -#[cfg(not(feature = "estimate-heapsize"))] -#[cfg(not(any(prefixed_jemalloc, target_os = "macos", target_os = "ios", target_os = "android", feature = "jemalloc-global")))] -impl MallocUnconditionalShallowSizeOf for Arc { - fn unconditional_shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize { - size_of::() - } -} - - #[cfg(feature = "std")] impl MallocUnconditionalSizeOf for Arc { fn unconditional_size_of(&self, ops: &mut MallocSizeOfOps) -> usize {