DRAFT: Adaptive evac with surge

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp

@@ -32,6 +32,58 @@
 #include "memory/allocation.hpp"
 #include "utilities/numberSeq.hpp"
 
+class ShenandoahPhaseTimeEstimator {
+ private:
+  static const uint MaxSamples = 8;
+
+  const char* _name;
+  bool   _changed;
+  bool   _changed_no_stdev;
+  uint   _first_index;
+  uint   _num_samples;
+  double _sum_of_x;
+  double _sum_of_y;
+  double _sum_of_xx;
+  double _sum_of_xy;
+  double _most_recent_start;
+  double _x_values[MaxSamples];
+  double _y_values[MaxSamples];
+  double _most_recent_prediction_x_value;
+  double _most_recent_prediction;
+  double _most_recent_prediction_x_value_no_stdev;
+  double _most_recent_prediction_no_stdev;
+  double _most_recent_bytes_allocated;
+
+ public:
+  explicit ShenandoahPhaseTimeEstimator(const char *name);
+
+  void add_sample(double independent_variable, double dependent_variable);
+
+  // Return conservative prediction of time required for next execution of this phase,
+  //   which is Max(average_prediction, linear_prediction),
+  //   average_prediction is average + std_dev, and
+  //   linear_prediction is determined best-fit line + std_dev of this calculation
+  double predict_at(double independent_value);
+
+  double predict_at_without_stdev(double independent_value);
+
+  void set_most_recent_start_time(double now) {
+    _most_recent_start = now;
+  }
+
+  double get_most_recent_start_time() {
+    return _most_recent_start;
+  }
+
+  void set_most_recent_bytes_allocated(size_t bytes) {
+    _most_recent_bytes_allocated = bytes;
+  }
+
+  size_t get_most_recent_bytes_allocated() {
+    return _most_recent_bytes_allocated;
+  }
+};
+
 class ShenandoahAllocationRate : public CHeapObj<mtGC> {
  public:
   explicit ShenandoahAllocationRate();
@@ -40,6 +92,7 @@
   double sample(size_t allocated);
 
   double upper_bound(double sds) const;
+  double average_rate(double sds) const;
   bool is_spiking(double rate, double threshold) const;
  private:
 
@@ -73,7 +126,7 @@
                                                      RegionData* data, size_t size,
                                                      size_t actual_free);
 
-  void record_cycle_start();
+  virtual void record_cycle_start();
   void record_success_concurrent();
   void record_success_degenerated();
   void record_success_full();
@@ -84,6 +137,17 @@
   virtual bool is_diagnostic()   { return false; }
   virtual bool is_experimental() { return false; }
 
+  virtual void record_phase_duration(ShenandoahMajorGCPhase p, double x, double duration) override;
+
+  void record_mark_end(double now, size_t marked_words) override;
+  // In non-generational mode, supply pip_words as zero
+  void record_evac_end(double now, size_t evacuated_words, size_t pip_words) override;
+  void record_update_end(double now, size_t updated_words) override;
+  // In non-generational mode, supply pip_words as zero
+  void record_final_roots_end(double now, size_t pip_words) override;
+
+  uint should_surge_phase(ShenandoahMajorGCPhase phase, double now) override;
+
  private:
   // These are used to adjust the margin of error and the spike threshold
   // in response to GC cycle outcomes. These values are shared, but the
@@ -99,6 +163,18 @@
 
   friend class ShenandoahAllocationRate;
 
+  void adjust_last_trigger_parameters(double amount);
+  void adjust_margin_of_error(double amount);
+  void adjust_spike_threshold(double amount);
+
+protected:
+  // Use this to estimate how much time will be required for future mark, evac, update, final-roots phases.
+  ShenandoahPhaseTimeEstimator _phase_stats[ShenandoahMajorGCPhase::_num_phases] = {
+    ShenandoahPhaseTimeEstimator("mark"),
+    ShenandoahPhaseTimeEstimator("evac"),
+    ShenandoahPhaseTimeEstimator("update"),
+    ShenandoahPhaseTimeEstimator("final-roots") };
+
   // Used to record the last trigger that signaled to start a GC.
   // This itself is used to decide whether or not to adjust the margin of
   // error for the average cycle time and allocation rate or the allocation
@@ -107,11 +183,6 @@
     SPIKE, RATE, OTHER
   };
 
-  void adjust_last_trigger_parameters(double amount);
-  void adjust_margin_of_error(double amount);
-  void adjust_spike_threshold(double amount);
-
-protected:
   ShenandoahAllocationRate _allocation_rate;
 
   // The margin of error expressed in standard deviations to add to our
@@ -140,6 +211,60 @@
   // source of feedback to adjust trigger parameters.
   TruncatedSeq _available;
 
+  // How many total words were evacuated in the most recently completed GC?
+  size_t _words_most_recently_evacuated;
+
+  // How many words do we expect to mark in the next GC?
+  // (aka how many words did we evacuate from most recently completed GC?)
+  size_t _anticipated_mark_words;
+
+  // How many words do we expect to evacuate in the next GC?
+  // (aka how many words did we evacuate from most recently completed GC?)
+  size_t _anticipated_evac_words;
+
+  // How many words do we expect to update in the next GC?
+  size_t _anticipated_update_words;
+
+  double predict_mark_time(size_t anticipated_marked_words) override;
+  double predict_evac_time(size_t anticipated_evac_words, size_t anticipated_pip_words) override;
+  double predict_update_time(size_t anticipated_update_words) override;
+  double predict_final_roots_time(size_t pip_words) override;
+
+  double predict_mark_time_nonconservative(size_t anticipated_marked_words) override;
+  double predict_evac_time_nonconservative(size_t anticipated_evac_words, size_t anticipated_pip_words) override;
+  double predict_update_time_nonconservative(size_t anticipated_update_words) override;
+  double predict_final_roots_time_nonconservative(size_t pip_words) override;
+
+  double predict_gc_time() override;
+  double predict_gc_time_nonconservative() override;
+
+  inline size_t get_anticipated_mark_words() {
+    return _anticipated_mark_words;
+  }
+
+  inline void set_anticipated_evac_words(size_t words) {
+#undef KELVIN_ANTICIPATION
+#ifdef KELVIN_ANTICIPATION
+    log_info(gc)("SAH::set_anticipated_evac_words(%zu)", words);
+#endif
+    _anticipated_evac_words = words;
+  }
+
+  inline size_t get_anticipated_evac_words() {
+    return _anticipated_evac_words;
+  }
+
+  inline void set_anticipated_update_words(size_t words) {
+#ifdef KELVIN_ANTICIPATION
+    log_info(gc)("SAH::set_anticipated_update_words(%zu)", words);
+#endif
+    _anticipated_update_words =  words;
+  }
+
+  inline size_t get_anticipated_update_words() {
+    return _anticipated_update_words;
+  }
+
   // A conservative minimum threshold of free space that we'll try to maintain when possible.
   // For example, we might trigger a concurrent gc if we are likely to drop below
   // this threshold, or we might consider this when dynamically resizing generations

src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp

@@ -31,6 +31,8 @@
 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
+#include "gc/shenandoah/shenandoahEvacInfo.hpp"
 #include "gc/shenandoah/shenandoahTrace.hpp"
 #include "logging/log.hpp"
 
@@ -58,6 +60,7 @@
   size_t preselected_candidates = 0;
 
   size_t total_garbage = 0;
+  size_t live_words_in_young = 0;
 
   size_t immediate_garbage = 0;
   size_t immediate_regions = 0;
@@ -69,8 +72,11 @@
 
   // This counts number of humongous regions that we intend to promote in this cycle.
   size_t humongous_regions_promoted = 0;
+  size_t humongous_live_words_promoted = 0;
   // This counts number of regular regions that will be promoted in place.
   size_t regular_regions_promoted_in_place = 0;
+  // This counts bytes of live memory within regular regions to be promoted in place.
+  size_t regular_regions_promoted_live = 0;
   // This counts bytes of memory used by regular regions to be promoted in place.
   size_t regular_regions_promoted_usage = 0;
   // This counts bytes of memory free in regular regions to be promoted in place.
@@ -85,6 +91,9 @@
     }
     size_t garbage = region->garbage();
     total_garbage += garbage;
+    if (region->is_young()) {
+      live_words_in_young += region->get_live_data_words();
+    }
     if (region->is_empty()) {
       free_regions++;
       free += region_size_bytes;
@@ -113,6 +122,7 @@
           if (region->get_top_before_promote() != nullptr) {
             // Region was included for promotion-in-place
             regular_regions_promoted_in_place++;
+	    regular_regions_promoted_live += region->get_live_data_words();
             regular_regions_promoted_usage += region->used_before_promote();
             regular_regions_promoted_free += region->free();
             regular_regions_promoted_garbage += region->garbage();
@@ -144,8 +154,10 @@
       } else {
         if (region->is_young() && region->age() >= tenuring_threshold) {
           oop obj = cast_to_oop(region->bottom());
-          size_t humongous_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize);
+	  size_t humongous_object_size = obj->size();
+          size_t humongous_regions = ShenandoahHeapRegion::required_regions(humongous_object_size * HeapWordSize);
           humongous_regions_promoted += humongous_regions;
+	  humongous_live_words_promoted += humongous_object_size;
         }
       }
     } else if (region->is_trash()) {
@@ -155,7 +167,13 @@
     }
   }
   heap->old_generation()->set_expected_humongous_region_promotions(humongous_regions_promoted);
+  heap->old_generation()->set_expected_promotable_humongous_region_live_data(humongous_live_words_promoted);
   heap->old_generation()->set_expected_regular_region_promotions(regular_regions_promoted_in_place);
+  heap->old_generation()->set_expected_promotable_regular_region_live_data(regular_regions_promoted_live);
+
+  ShenandoahYoungHeuristics* young_heuristics = heap->young_generation()->heuristics();
+  young_heuristics->set_young_live_words_after_most_recent_mark(live_words_in_young);
+
   log_info(gc, ergo)("Planning to promote in place %zu humongous regions and %zu"
                      " regular regions, spanning a total of %zu used bytes",
                      humongous_regions_promoted, regular_regions_promoted_in_place,
@@ -175,12 +193,24 @@
   bool doing_promote_in_place = (humongous_regions_promoted + regular_regions_promoted_in_place > 0);
   if (doing_promote_in_place || (preselected_candidates > 0) || (immediate_percent <= ShenandoahImmediateThreshold)) {
     // Only young collections need to prime the collection set.
+    bool need_to_finalize_mixed = false;
     if (_generation->is_young()) {
-      heap->old_generation()->heuristics()->prime_collection_set(collection_set);
+      need_to_finalize_mixed = heap->old_generation()->heuristics()->prime_collection_set(collection_set);
     }
 
     // Call the subclasses to add young-gen regions into the collection set.
     choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free);
+
+    if (_generation->is_young()) {
+      // Especially when young-gen trigger is expedited in order to finish mixed evacuations, there may not be
+      // enough consolidated garbage to make effective use of young-gen evacuation reserve.  If there is still
+      // young-gen reserve available following selection of the young-gen collection set, see if we can use
+      // this memory to expand the old-gen evacuation collection set.
+      need_to_finalize_mixed |= heap->old_generation()->heuristics()->top_off_collection_set();
+      if (need_to_finalize_mixed) {
+        heap->old_generation()->heuristics()->finalize_mixed_evacs();
+      }
+    }
   }
 
   if (collection_set->has_old_regions()) {
@@ -240,6 +270,51 @@
 
     ShenandoahTracer().report_evacuation_info(&evacInfo);
   }
+
+#undef KELVIN_CSET
+#ifdef KELVIN_CSET
+  log_info(gc)("Finished choosing collection set. Here is what we know:");
+
+  size_t young_bytes_evacuated = collection_set->get_young_bytes_reserved_for_evacuation();
+  log_info(gc)(" Bytes to be evacuated from young: %zu", young_bytes_evacuated);
+
+  size_t young_evac_garbage = collection_set->garbage() - collection_set->get_old_garbage();
+  log_info(gc)("  reclaiming %zu bytes of young garbage", young_evac_garbage);
+
+  log_info(gc)(" Bytes to be promoted in place from regular young: %zu in %zu regions",
+	       regular_regions_promoted_usage, regular_regions_promoted_in_place);
+  log_info(gc)("  of which %zu is live, %zu is garbage, and %zu was already free",
+	       regular_regions_promoted_usage - regular_regions_promoted_garbage, regular_regions_promoted_garbage,
+	       regular_regions_promoted_free);
+
+  log_info(gc)(" Bytes to be promoted in place from humongus young including waste: %zu in %zu regions",
+	       humongous_regions_promoted * region_size_bytes, humongous_regions_promoted);
+  log_info(gc)(" young_generation used including waste: %zu", heap->young_generation()->used_including_humongous_waste());
+
+  size_t young_update = heap->young_generation()->used_including_humongous_waste() - 
+    (regular_regions_promoted_usage + humongous_regions_promoted * region_size_bytes + young_evac_garbage + young_bytes_evacuated);
+  log_info(gc)(" Young bytes to be updated is young used minus (evacuated and promoted in place and reclaimed garbage: %zu",
+	       young_update);
+
+  if (collection_set->has_old_regions()) {
+    log_info(gc)(" This is a mixed evacuation, so we'll need to update all of old that is not in cset");
+
+    size_t old_bytes_evacuated = collection_set->get_old_bytes_reserved_for_evacuation();
+    log_info(gc)(" Bytes to be evacuated from old: %zu", old_bytes_evacuated);
+
+    size_t old_evac_garbage = collection_set->get_old_garbage();
+    log_info(gc)("  reclaiming %zu bytes of old garbage", young_evac_garbage);
+
+    size_t old_usage = heap->old_generation()->used_including_humongous_waste() +
+      (regular_regions_promoted_usage + humongous_regions_promoted * region_size_bytes);
+    size_t old_update = old_usage - (old_bytes_evacuated + old_evac_garbage);
+
+    log_info(gc)(" Bytes to be updated from old: %zu", old_update);
+
+  } else {
+    log_info(gc)(" This is not a mixed evacuation.  Assume old update is same as words scanned for mark remset scan");
+  }
+#endif
 }