From 89b305ca49fa64ac539aa2938c5a6ad0e3cb3741 Mon Sep 17 00:00:00 2001 From: Axel Boldt-Christmas Date: Wed, 21 Jun 2023 15:32:06 +0000 Subject: [PATCH] 8367972: ZGC: Reduce ZBarrierSet includes --- src/hotspot/share/gc/z/zBarrierSet.cpp | 107 +++++++++++++++--- src/hotspot/share/gc/z/zBarrierSet.hpp | 19 +++- src/hotspot/share/gc/z/zBarrierSet.inline.hpp | 74 ++++-------- src/hotspot/share/gc/z/zObjArrayAllocator.cpp | 1 + .../share/gc/z/zPhysicalMemoryManager.cpp | 1 + .../share/gc/z/zRangeRegistry.inline.hpp | 1 + src/hotspot/share/precompiled/precompiled.hpp | 1 - src/hotspot/share/prims/whitebox.cpp | 1 + src/hotspot/share/runtime/stackValue.cpp | 3 - .../hotspot/gtest/runtime/test_os_windows.cpp | 2 +- 10 files changed, 137 insertions(+), 73 deletions(-) diff --git a/src/hotspot/share/gc/z/zBarrierSet.cpp b/src/hotspot/share/gc/z/zBarrierSet.cpp index c71c404712ca1..23e64def69ddc 100644 --- a/src/hotspot/share/gc/z/zBarrierSet.cpp +++ b/src/hotspot/share/gc/z/zBarrierSet.cpp @@ -21,6 +21,8 @@ * questions. */ +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetNMethod.hpp" @@ -30,6 +32,7 @@ #include "gc/z/zHeap.inline.hpp" #include "gc/z/zStackWatermark.hpp" #include "gc/z/zThreadLocalData.hpp" +#include "runtime/atomicAccess.hpp" #include "runtime/deoptimization.hpp" #include "runtime/frame.inline.hpp" #include "runtime/javaThread.hpp" @@ -46,6 +49,96 @@ class ZBarrierSetC1; class ZBarrierSetC2; +class ZColorStoreGoodOopClosure : public BasicOopIterateClosure { +public: + virtual void do_oop(oop* p_) { + volatile zpointer* const p = (volatile zpointer*)p_; + const zpointer ptr = ZBarrier::load_atomic(p); + const zaddress addr = ZPointer::uncolor(ptr); + AtomicAccess::store(p, ZAddress::store_good(addr)); + } + + virtual void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +class ZLoadBarrierOopClosure : public BasicOopIterateClosure { +public: + virtual void do_oop(oop* p) { + ZBarrier::load_barrier_on_oop_field((zpointer*)p); + } + + virtual void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +void ZBarrierSet::load_barrier_all(oop src, size_t size) { + check_is_valid_zaddress(src); + + ZLoadBarrierOopClosure cl; + ZIterator::oop_iterate(src, &cl); +} + +void ZBarrierSet::color_store_good_all(oop dst, size_t size) { + check_is_valid_zaddress(dst); + assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects"); + + ZColorStoreGoodOopClosure cl_sg; + ZIterator::oop_iterate(dst, &cl_sg); +} + +zaddress ZBarrierSet::load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o) { + return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); +} + +zaddress ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) { + return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); +} + +zaddress ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) { + return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); +} + +zaddress ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) { + return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); +} + +zaddress ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) { + return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); +} + +void ZBarrierSet::store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal) { + ZBarrier::store_barrier_on_heap_oop_field(p, heal); +} + +void ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p) { + ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p); +} + +void ZBarrierSet::store_barrier_on_native_oop_field(volatile zpointer* p, bool heal) { + ZBarrier::store_barrier_on_native_oop_field(p, heal); +} + +zaddress ZBarrierSet::load_barrier_on_oop_field(volatile zpointer* p) { + return ZBarrier::load_barrier_on_oop_field(p); +} + +void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) { + volatile zpointer* src = (volatile zpointer*)src_obj->base(); + volatile zpointer* dst = (volatile zpointer*)dst_obj->base(); + const int length = src_obj->length(); + + for (const volatile zpointer* const end = src + length; src < end; src++, dst++) { + zaddress elem = ZBarrier::load_barrier_on_oop_field(src); + // We avoid healing here because the store below colors the pointer store good, + // hence avoiding the cost of a CAS. + ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */); + AtomicAccess::store(dst, ZAddress::store_good(elem)); + } +} + ZBarrierSet::ZBarrierSet() : BarrierSet(make_barrier_set_assembler(), make_barrier_set_c1(), @@ -153,20 +246,6 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) { deoptimize_allocation(thread); } -void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) { - volatile zpointer* src = (volatile zpointer*)src_obj->base(); - volatile zpointer* dst = (volatile zpointer*)dst_obj->base(); - const int length = src_obj->length(); - - for (const volatile zpointer* const end = src + length; src < end; src++, dst++) { - zaddress elem = ZBarrier::load_barrier_on_oop_field(src); - // We avoid healing here because the store below colors the pointer store good, - // hence avoiding the cost of a CAS. - ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */); - AtomicAccess::store(dst, ZAddress::store_good(elem)); - } -} - void ZBarrierSet::print_on(outputStream* st) const { st->print_cr("ZBarrierSet"); } diff --git a/src/hotspot/share/gc/z/zBarrierSet.hpp b/src/hotspot/share/gc/z/zBarrierSet.hpp index 51eb16319a0ca..502cc36d380b9 100644 --- a/src/hotspot/share/gc/z/zBarrierSet.hpp +++ b/src/hotspot/share/gc/z/zBarrierSet.hpp @@ -33,14 +33,29 @@ class ZBarrierSet : public BarrierSet { private: static zpointer store_good(oop obj); + static void load_barrier_all(oop src, size_t size); + static void color_store_good_all(oop dst, size_t size); + + static zaddress load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o); + static zaddress no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o); + static zaddress no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o); + static zaddress load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o); + static zaddress load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o); + + static void store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal); + static void no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p); + static void store_barrier_on_native_oop_field(volatile zpointer* p, bool heal); + + static zaddress load_barrier_on_oop_field(volatile zpointer* p); + + static void clone_obj_array(objArrayOop src, objArrayOop dst); + public: ZBarrierSet(); static ZBarrierSetAssembler* assembler(); static bool barrier_needed(DecoratorSet decorators, BasicType type); - static void clone_obj_array(objArrayOop src, objArrayOop dst); - virtual void on_thread_create(Thread* thread); virtual void on_thread_destroy(Thread* thread); virtual void on_thread_attach(Thread* thread); diff --git a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp index 16f2a303cb2e5..e27933d1f7b21 100644 --- a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp @@ -28,10 +28,9 @@ #include "gc/shared/accessBarrierSupport.inline.hpp" #include "gc/z/zAddress.inline.hpp" -#include "gc/z/zBarrier.inline.hpp" -#include "gc/z/zIterator.inline.hpp" +#include "gc/z/zHeap.hpp" #include "gc/z/zNMethod.hpp" -#include "memory/iterator.inline.hpp" +#include "oops/objArrayOop.hpp" #include "utilities/debug.hpp" template @@ -68,21 +67,21 @@ inline zaddress ZBarrierSet::AccessBarrier::load_barrie if (HasDecorator::value) { if (HasDecorator::value) { // Load barriers on strong oop refs don't keep objects alive - return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o); } else if (HasDecorator::value) { - return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); + return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); } else { assert((HasDecorator::value), "Must be"); - return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); + return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); } } else { if (HasDecorator::value) { - return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o); } else if (HasDecorator::value) { - return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o); } else { assert((HasDecorator::value), "Must be"); - return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o); } } } @@ -97,21 +96,21 @@ inline zaddress ZBarrierSet::AccessBarrier::load_barrie if (HasDecorator::value) { if (decorators_known_strength & ON_STRONG_OOP_REF) { // Load barriers on strong oop refs don't keep objects alive - return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o); } else if (decorators_known_strength & ON_WEAK_OOP_REF) { - return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); + return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); } else { assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be"); - return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); + return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); } } else { if (decorators_known_strength & ON_STRONG_OOP_REF) { - return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o); } else if (decorators_known_strength & ON_WEAK_OOP_REF) { - return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o); } else { assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be"); - return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); + return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o); } } } @@ -126,7 +125,7 @@ inline zpointer ZBarrierSet::store_good(oop obj) { template inline void ZBarrierSet::AccessBarrier::store_barrier_heap_with_healing(zpointer* p) { if (!HasDecorator::value) { - ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */); + ZBarrierSet::store_barrier_on_heap_oop_field(p, true /* heal */); } else { assert(false, "Should not be used on uninitialized memory"); } @@ -135,21 +134,21 @@ inline void ZBarrierSet::AccessBarrier::store_barrier_h template inline void ZBarrierSet::AccessBarrier::store_barrier_heap_without_healing(zpointer* p) { if (!HasDecorator::value) { - ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */); + ZBarrierSet::store_barrier_on_heap_oop_field(p, false /* heal */); } } template inline void ZBarrierSet::AccessBarrier::no_keep_alive_store_barrier_heap(zpointer* p) { if (!HasDecorator::value) { - ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p); + ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(p); } } template inline void ZBarrierSet::AccessBarrier::store_barrier_native_with_healing(zpointer* p) { if (!HasDecorator::value) { - ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */); + ZBarrierSet::store_barrier_on_native_oop_field(p, true /* heal */); } else { assert(false, "Should not be used on uninitialized memory"); } @@ -158,7 +157,7 @@ inline void ZBarrierSet::AccessBarrier::store_barrier_n template inline void ZBarrierSet::AccessBarrier::store_barrier_native_without_healing(zpointer* p) { if (!HasDecorator::value) { - ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */); + ZBarrierSet::store_barrier_on_native_oop_field(p, false /* heal */); } } @@ -325,7 +324,7 @@ template inline zaddress ZBarrierSet::AccessBarrier::oop_copy_one_barriers(zpointer* dst, zpointer* src) { store_barrier_heap_without_healing(dst); - return ZBarrier::load_barrier_on_oop_field(src); + return ZBarrierSet::load_barrier_on_oop_field(src); } template @@ -403,31 +402,6 @@ inline bool ZBarrierSet::AccessBarrier::oop_arraycopy_i return oop_arraycopy_in_heap_no_check_cast(dst, src, length); } -class ZColorStoreGoodOopClosure : public BasicOopIterateClosure { -public: - virtual void do_oop(oop* p_) { - volatile zpointer* const p = (volatile zpointer*)p_; - const zpointer ptr = ZBarrier::load_atomic(p); - const zaddress addr = ZPointer::uncolor(ptr); - AtomicAccess::store(p, ZAddress::store_good(addr)); - } - - virtual void do_oop(narrowOop* p) { - ShouldNotReachHere(); - } -}; - -class ZLoadBarrierOopClosure : public BasicOopIterateClosure { -public: - virtual void do_oop(oop* p) { - ZBarrier::load_barrier_on_oop_field((zpointer*)p); - } - - virtual void do_oop(narrowOop* p) { - ShouldNotReachHere(); - } -}; - template inline void ZBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) { check_is_valid_zaddress(src); @@ -444,17 +418,13 @@ inline void ZBarrierSet::AccessBarrier::clone_in_heap(o } // Fix the oops - ZLoadBarrierOopClosure cl; - ZIterator::oop_iterate(src, &cl); + ZBarrierSet::load_barrier_all(src, size); // Clone the object Raw::clone_in_heap(src, dst, size); - assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects"); - // Color store good before handing out - ZColorStoreGoodOopClosure cl_sg; - ZIterator::oop_iterate(dst, &cl_sg); + ZBarrierSet::color_store_good_all(dst, size); } // diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp index ddb0ca4927849..e3e364d5cd29a 100644 --- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp +++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp @@ -21,6 +21,7 @@ * questions. */ +#include "gc/z/zGeneration.inline.hpp" #include "gc/z/zObjArrayAllocator.hpp" #include "gc/z/zThreadLocalData.hpp" #include "gc/z/zUtils.inline.hpp" diff --git a/src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp b/src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp index 54477d19a2700..1a38efb89fd5b 100644 --- a/src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp +++ b/src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp @@ -25,6 +25,7 @@ #include "gc/z/zAddress.inline.hpp" #include "gc/z/zArray.inline.hpp" #include "gc/z/zGlobals.hpp" +#include "gc/z/zGranuleMap.inline.hpp" #include "gc/z/zLargePages.inline.hpp" #include "gc/z/zList.inline.hpp" #include "gc/z/zNMT.hpp" diff --git a/src/hotspot/share/gc/z/zRangeRegistry.inline.hpp b/src/hotspot/share/gc/z/zRangeRegistry.inline.hpp index de34aca07c4f1..e466cc764b4b9 100644 --- a/src/hotspot/share/gc/z/zRangeRegistry.inline.hpp +++ b/src/hotspot/share/gc/z/zRangeRegistry.inline.hpp @@ -27,6 +27,7 @@ #include "gc/z/zRangeRegistry.hpp" #include "gc/z/zAddress.inline.hpp" +#include "gc/z/zArray.inline.hpp" #include "gc/z/zList.inline.hpp" #include "gc/z/zLock.inline.hpp" diff --git a/src/hotspot/share/precompiled/precompiled.hpp b/src/hotspot/share/precompiled/precompiled.hpp index 670bd55e423d8..1f503a9674947 100644 --- a/src/hotspot/share/precompiled/precompiled.hpp +++ b/src/hotspot/share/precompiled/precompiled.hpp @@ -45,7 +45,6 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #endif #if INCLUDE_ZGC -#include "gc/z/zBarrier.inline.hpp" #include "gc/z/zGeneration.inline.hpp" #include "gc/z/zHeap.inline.hpp" #endif diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp index e8e873fcb6653..6d9ab57bb9ac4 100644 --- a/src/hotspot/share/prims/whitebox.cpp +++ b/src/hotspot/share/prims/whitebox.cpp @@ -120,6 +120,7 @@ #endif // INCLUDE_SERIALGC #if INCLUDE_ZGC #include "gc/z/zAddress.inline.hpp" +#include "gc/z/zHeap.inline.hpp" #endif // INCLUDE_ZGC #if INCLUDE_JVMCI #include "jvmci/jvmciEnv.hpp" diff --git a/src/hotspot/share/runtime/stackValue.cpp b/src/hotspot/share/runtime/stackValue.cpp index 5e01efc162240..52d2631f3c94f 100644 --- a/src/hotspot/share/runtime/stackValue.cpp +++ b/src/hotspot/share/runtime/stackValue.cpp @@ -30,9 +30,6 @@ #include "runtime/globals.hpp" #include "runtime/handles.inline.hpp" #include "runtime/stackValue.hpp" -#if INCLUDE_ZGC -#include "gc/z/zBarrier.inline.hpp" -#endif #if INCLUDE_SHENANDOAHGC #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" #endif diff --git a/test/hotspot/gtest/runtime/test_os_windows.cpp b/test/hotspot/gtest/runtime/test_os_windows.cpp index 9fe0de555156e..ad2708480770b 100644 --- a/test/hotspot/gtest/runtime/test_os_windows.cpp +++ b/test/hotspot/gtest/runtime/test_os_windows.cpp @@ -27,7 +27,7 @@ #include "logging/log.hpp" #include "runtime/flags/flagSetting.hpp" #include "runtime/globals_extension.hpp" -#include "runtime/os.hpp" +#include "runtime/os.inline.hpp" #include "concurrentTestRunner.inline.hpp" #include "unittest.hpp"