Merge branch 'upstream-master'

Author: Datadog Syncup Service

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -35,8 +35,9 @@
 #include "gc/g1/g1Trace.hpp"
 #include "gc/g1/g1YoungGCAllocationFailureInjector.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/partialArraySplitter.inline.hpp"
 #include "gc/shared/partialArrayState.hpp"
-#include "gc/shared/partialArrayTaskStepper.inline.hpp"
+#include "gc/shared/partialArrayTaskStats.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "memory/allocation.inline.hpp"
@@ -80,8 +81,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
     _surviving_young_words(nullptr),
     _surviving_words_length(collection_set->young_region_length() + 1),
     _old_gen_is_full(false),
-    _partial_array_state_allocator(g1h->partial_array_state_manager()),
-    _partial_array_stepper(num_workers, ParGCArrayScanChunk),
+    _partial_array_splitter(g1h->partial_array_state_manager(), num_workers),
     _string_dedup_requests(),
     _max_num_optional_regions(collection_set->optional_region_length()),
     _numa(g1h->numa()),
@@ -169,9 +169,12 @@ void G1ParScanThreadState::verify_task(oop* task) const {
 }
 
 void G1ParScanThreadState::verify_task(PartialArrayState* task) const {
-  // Must be in the collection set--it's already been copied.
-  oop p = task->source();
-  assert(_g1h->is_in_cset(p), "p=" PTR_FORMAT, p2i(p));
+  assert(task != nullptr, "invariant");
+  // Source isn't used for processing, so not recorded in task.
+  assert(task->source() == nullptr, "invariant");
+  oop p = task->destination();
+  assert(_g1h->is_in_reserved(p),
+         "task=" PTR_FORMAT " dest=" PTR_FORMAT, p2i(task), p2i(p));
 }
 
 void G1ParScanThreadState::verify_task(ScannerTask task) const {
@@ -222,38 +225,17 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
 }
 
 MAYBE_INLINE_EVACUATION
-void G1ParScanThreadState::do_partial_array(PartialArrayState* state) {
-  oop to_obj = state->destination();
-
-#ifdef ASSERT
-  oop from_obj = state->source();
-  assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
-  assert(from_obj->is_forwarded(), "must be forwarded");
-  assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
-  assert(to_obj->is_objArray(), "must be obj array");
-#endif // ASSERT
-
-  objArrayOop to_array = objArrayOop(to_obj);
-
-  // Claim a chunk and get number of additional tasks to enqueue.
-  PartialArrayTaskStepper::Step step = _partial_array_stepper.next(state);
-  // Push any additional partial scan tasks needed.  Pushed before processing
-  // the claimed chunk to allow other workers to steal while we're processing.
-  if (step._ncreate > 0) {
-    state->add_references(step._ncreate);
-    for (uint i = 0; i < step._ncreate; ++i) {
-      push_on_queue(ScannerTask(state));
-    }
-  }
-
+void G1ParScanThreadState::do_partial_array(PartialArrayState* state, bool stolen) {
+  // Access state before release by claim().
+  objArrayOop to_array = objArrayOop(state->destination());
+  PartialArraySplitter::Claim claim =
+    _partial_array_splitter.claim(state, _task_queue, stolen);
   G1HeapRegionAttr dest_attr = _g1h->region_attr(to_array);
   G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_new_survivor());
   // Process claimed task.
   to_array->oop_iterate_range(&_scanner,
-                              checked_cast<int>(step._index),
-                              checked_cast<int>(step._index + _partial_array_stepper.chunk_size()));
-  // Release reference to the state, now that we're done with it.
-  _partial_array_state_allocator.release(state);
+                              checked_cast<int>(claim._start),
+                              checked_cast<int>(claim._end));
 }
 
 MAYBE_INLINE_EVACUATION
@@ -265,27 +247,10 @@ void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
   assert(to_obj->is_objArray(), "precondition");
 
   objArrayOop to_array = objArrayOop(to_obj);
-
   size_t array_length = to_array->length();
-  PartialArrayTaskStepper::Step step = _partial_array_stepper.start(array_length);
-
-  // Push any needed partial scan tasks.  Pushed before processing the
-  // initial chunk to allow other workers to steal while we're processing.
-  if (step._ncreate > 0) {
-    assert(step._index < array_length, "invariant");
-    assert(((array_length - step._index) % _partial_array_stepper.chunk_size()) == 0,
-           "invariant");
-    PartialArrayState* state =
-      _partial_array_state_allocator.allocate(from_obj, to_obj,
-                                              step._index,
-                                              array_length,
-                                              step._ncreate);
-    for (uint i = 0; i < step._ncreate; ++i) {
-      push_on_queue(ScannerTask(state));
-    }
-  } else {
-    assert(step._index == array_length, "invariant");
-  }
+  size_t initial_chunk_size =
+    // The source array is unused when processing states.
+    _partial_array_splitter.start(_task_queue, nullptr, to_array, array_length);
 
   // Skip the card enqueue iff the object (to_array) is in survivor region.
   // However, G1HeapRegion::is_survivor() is too expensive here.
@@ -296,18 +261,18 @@ void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
   // Process the initial chunk.  No need to process the type in the
   // klass, as it will already be handled by processing the built-in
   // module.
-  to_array->oop_iterate_range(&_scanner, 0, checked_cast<int>(step._index));
+  to_array->oop_iterate_range(&_scanner, 0, checked_cast<int>(initial_chunk_size));
 }
 
 MAYBE_INLINE_EVACUATION
-void G1ParScanThreadState::dispatch_task(ScannerTask task) {
+void G1ParScanThreadState::dispatch_task(ScannerTask task, bool stolen) {
   verify_task(task);
   if (task.is_narrow_oop_ptr()) {
     do_oop_evac(task.to_narrow_oop_ptr());
   } else if (task.is_oop_ptr()) {
     do_oop_evac(task.to_oop_ptr());
   } else {
-    do_partial_array(task.to_partial_array_state());
+    do_partial_array(task.to_partial_array_state(), stolen);
   }
 }
 
@@ -320,11 +285,11 @@ void G1ParScanThreadState::trim_queue_to_threshold(uint threshold) {
   do {
     while (_task_queue->pop_overflow(task)) {
       if (!_task_queue->try_push_to_taskqueue(task)) {
-        dispatch_task(task);
+        dispatch_task(task, false);
       }
     }
     while (_task_queue->pop_local(task, threshold)) {
-      dispatch_task(task);
+      dispatch_task(task, false);
     }
   } while (!_task_queue->overflow_empty());
 }
@@ -333,7 +298,7 @@ ATTRIBUTE_FLATTEN
 void G1ParScanThreadState::steal_and_trim_queue(G1ScannerTasksQueueSet* task_queues) {
   ScannerTask stolen_task;
   while (task_queues->steal(_worker_id, stolen_task)) {
-    dispatch_task(stolen_task);
+    dispatch_task(stolen_task, true);
     // Processing stolen task may have added tasks to our queue.
     trim_queue();
   }
@@ -717,6 +682,14 @@ void G1ParScanThreadState::update_numa_stats(uint node_index) {
   }
 }
 
+#if TASKQUEUE_STATS
+
+PartialArrayTaskStats* G1ParScanThreadState::partial_array_task_stats() {
+  return _partial_array_splitter.stats();
+}
+
+#endif // TASKQUEUE_STATS
+
 G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
                                                  uint num_workers,
                                                  G1CollectionSet* collection_set,
@@ -744,3 +717,15 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
   FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
   FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers);
 }
+
+#if TASKQUEUE_STATS
+
+void G1ParScanThreadStateSet::print_partial_array_task_stats() {
+  auto get_stats = [&](uint i) {
+    return state_for_worker(i)->partial_array_task_stats();
+  };
+  PartialArrayTaskStats::log_set(_num_workers, get_stats,
+                                 "Partial Array Task Stats");
+}
+
+#endif // TASKQUEUE_STATS

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -32,8 +32,8 @@
 #include "gc/shared/ageTable.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/partialArraySplitter.hpp"
 #include "gc/shared/partialArrayState.hpp"
-#include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/allocation.hpp"
@@ -84,8 +84,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // Indicates whether in the last generation (old) there is no more space
   // available for allocation.
   bool _old_gen_is_full;
-  PartialArrayStateAllocator _partial_array_state_allocator;
-  PartialArrayTaskStepper _partial_array_stepper;
+  PartialArraySplitter _partial_array_splitter;
   StringDedup::Requests _string_dedup_requests;
 
   G1CardTable* ct() { return _ct; }
@@ -163,8 +162,12 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // HeapWords copied.
   size_t flush_stats(size_t* surviving_young_words, uint num_workers, BufferNodeList* buffer_log);
 
+#if TASKQUEUE_STATS
+  PartialArrayTaskStats* partial_array_task_stats();
+#endif // TASKQUEUE_STATS
+
 private:
-  void do_partial_array(PartialArrayState* state);
+  void do_partial_array(PartialArrayState* state, bool stolen);
   void start_partial_objarray(G1HeapRegionAttr dest_dir, oop from, oop to);
 
   HeapWord* allocate_copy_slow(G1HeapRegionAttr* dest_attr,
@@ -187,7 +190,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // This method is applied to the fields of the objects that have just been copied.
   template <class T> void do_oop_evac(T* p);
 
-  void dispatch_task(ScannerTask task);
+  void dispatch_task(ScannerTask task, bool stolen);
 
   // Tries to allocate word_sz in the PLAB of the next "generation" after trying to
   // allocate into dest. Previous_plab_refill_failed indicates whether previous
@@ -259,6 +262,9 @@ class G1ParScanThreadStateSet : public StackObj {
 
   void flush_stats();
   void record_unused_optional_region(G1HeapRegion* hr);
+#if TASKQUEUE_STATS
+  void print_partial_array_task_stats();
+#endif // TASKQUEUE_STATS
 
   G1ParScanThreadState* state_for_worker(uint worker_id);
   uint num_workers() const { return _num_workers; }

src/hotspot/share/gc/g1/g1YoungCollector.cpp

@@ -1013,6 +1013,12 @@ void G1YoungCollector::post_evacuate_collection_set(G1EvacInfo* evacuation_info,
 
   allocator()->release_gc_alloc_regions(evacuation_info);
 
+#if TASKQUEUE_STATS
+  // Logging uses thread states, which are deleted by cleanup, so this must
+  // be done before cleanup.
+  per_thread_states->print_partial_array_task_stats();
+#endif // TASKQUEUE_STATS
+
   post_evacuate_cleanup_1(per_thread_states);
 
   post_evacuate_cleanup_2(per_thread_states, evacuation_info);