Add aggregated memory histograms.

This introduces V8.MemoryHeapCommitted and V8.MemoryHeapUsed histograms.

In contrast to the existing memory histograms, the new histograms are uniform in time, i.e. their samples happen at regular time intervals. The --histogram-interval specifies the length of the interval.

We implement this by linearly interpolating memory stats between GC and idle notification events.

BUG=chromium:485472
LOG=NO

Review URL: https://codereview.chromium.org/1125683004

Cr-Commit-Position: refs/heads/master@{#28292}

src/counters.cc

@@ -79,8 +79,25 @@ Counters::Counters(Isolate* isolate) {
     HISTOGRAM_PERCENTAGE_LIST(HP)
 #undef HP
 
+
+// Exponential histogram assigns bucket limits to points
+// p[1], p[2], ... p[n] such that p[i+1] / p[i] = constant.
+// The constant factor is equal to the n-th root of (high / low),
+// where the n is the number of buckets, the low is the lower limit,
+// the high is the upper limit.
+// For n = 50, low = 1000, high = 500000: the factor = 1.13.
 #define HM(name, caption) \
     name##_ = Histogram(#caption, 1000, 500000, 50, isolate);
+  HISTOGRAM_LEGACY_MEMORY_LIST(HM)
+#undef HM
+// For n = 100, low = 4000, high = 2000000: the factor = 1.06.
+#define HM(name, caption) \
+  name##_ = Histogram(#caption, 4000, 2000000, 100, isolate);
+  HISTOGRAM_MEMORY_LIST(HM)
+#undef HM
+
+#define HM(name, caption) \
+  aggregated_##name##_ = AggregatedMemoryHistogram<Histogram>(&name##_);
     HISTOGRAM_MEMORY_LIST(HM)
 #undef HM
 
@@ -173,7 +190,7 @@ void Counters::ResetHistograms() {
 #undef HP
 
 #define HM(name, caption) name##_.Reset();
-    HISTOGRAM_MEMORY_LIST(HM)
+    HISTOGRAM_LEGACY_MEMORY_LIST(HM)
 #undef HM
 }
 

src/counters.h

@@ -358,6 +358,124 @@ class AggregatedHistogramTimerScope {
 };
 
 
+// AggretatedMemoryHistogram collects (time, value) sample pairs and turns
+// them into time-uniform samples for the backing historgram, such that the
+// backing histogram receives one sample every T ms, where the T is controlled
+// by the FLAG_histogram_interval.
+//
+// More formally: let F be a real-valued function that maps time to sample
+// values. We define F as a linear interpolation between adjacent samples. For
+// each time interval [x; x + T) the backing histogram gets one sample value
+// that is the average of F(t) in the interval.
+template <typename Histogram>
+class AggregatedMemoryHistogram {
+ public:
+  AggregatedMemoryHistogram()
+      : is_initialized_(false),
+        start_ms_(0.0),
+        last_ms_(0.0),
+        aggregate_value_(0.0),
+        last_value_(0.0),
+        backing_histogram_(NULL) {}
+
+  explicit AggregatedMemoryHistogram(Histogram* backing_histogram)
+      : AggregatedMemoryHistogram() {
+    backing_histogram_ = backing_histogram;
+  }
+
+  // Invariants that hold before and after AddSample if
+  // is_initialized_ is true:
+  //
+  // 1) For we processed samples that came in before start_ms_ and sent the
+  // corresponding aggregated samples to backing histogram.
+  // 2) (last_ms_, last_value_) is the last received sample.
+  // 3) last_ms_ < start_ms_ + FLAG_histogram_interval.
+  // 4) aggregate_value_ is the average of the function that is constructed by
+  // linearly interpolating samples received between start_ms_ and last_ms_.
+  void AddSample(double current_ms, double current_value);
+
+ private:
+  double Aggregate(double current_ms, double current_value);
+  bool is_initialized_;
+  double start_ms_;
+  double last_ms_;
+  double aggregate_value_;
+  double last_value_;
+  Histogram* backing_histogram_;
+};
+
+
+template <typename Histogram>
+void AggregatedMemoryHistogram<Histogram>::AddSample(double current_ms,
+                                                     double current_value) {
+  if (!is_initialized_) {
+    aggregate_value_ = current_value;
+    start_ms_ = current_ms;
+    last_value_ = current_value;
+    last_ms_ = current_ms;
+    is_initialized_ = true;
+  } else {
+    const double kEpsilon = 1e-6;
+    const int kMaxSamples = 1000;
+    if (current_ms < last_ms_ + kEpsilon) {
+      // Two samples have the same time, remember the last one.
+      last_value_ = current_value;
+    } else {
+      double sample_interval_ms = FLAG_histogram_interval;
+      double end_ms = start_ms_ + sample_interval_ms;
+      if (end_ms <= current_ms + kEpsilon) {
+        // Linearly interpolate between the last_ms_ and the current_ms.
+        double slope = (current_value - last_value_) / (current_ms - last_ms_);
+        int i;
+        // Send aggregated samples to the backing histogram from the start_ms
+        // to the current_ms.
+        for (i = 0; i < kMaxSamples && end_ms <= current_ms + kEpsilon; i++) {
+          double end_value = last_value_ + (end_ms - last_ms_) * slope;
+          double sample_value;
+          if (i == 0) {
+            // Take aggregate_value_ into account.
+            sample_value = Aggregate(end_ms, end_value);
+          } else {
+            // There is no aggregate_value_ for i > 0.
+            sample_value = (last_value_ + end_value) / 2;
+          }
+          backing_histogram_->AddSample(static_cast<int>(sample_value + 0.5));
+          last_value_ = end_value;
+          last_ms_ = end_ms;
+          end_ms += sample_interval_ms;
+        }
+        if (i == kMaxSamples) {
+          // We hit the sample limit, ignore the remaining samples.
+          aggregate_value_ = current_value;
+          start_ms_ = current_ms;
+        } else {
+          aggregate_value_ = last_value_;
+          start_ms_ = last_ms_;
+        }
+      }
+      aggregate_value_ = current_ms > start_ms_ + kEpsilon
+                             ? Aggregate(current_ms, current_value)
+                             : aggregate_value_;
+      last_value_ = current_value;
+      last_ms_ = current_ms;
+    }
+  }
+}
+
+
+template <typename Histogram>
+double AggregatedMemoryHistogram<Histogram>::Aggregate(double current_ms,
+                                                       double current_value) {
+  double interval_ms = current_ms - start_ms_;
+  double value = (current_value + last_value_) / 2;
+  // The aggregate_value_ is the average for [start_ms_; last_ms_].
+  // The value is the average for [last_ms_; current_ms].
+  // Return the weighted average of the aggregate_value_ and the value.
+  return aggregate_value_ * ((last_ms_ - start_ms_) / interval_ms) +
+         value * ((current_ms - last_ms_) / interval_ms);
+}
+
+
 #define HISTOGRAM_RANGE_LIST(HR)                                              \
   /* Generic range histograms */                                              \
   HR(detached_context_age_in_gc, V8.DetachedContextAgeInGC, 0, 20, 21)        \
@@ -414,15 +532,16 @@ class AggregatedHistogramTimerScope {
   HP(codegen_fraction_crankshaft, V8.CodegenFractionCrankshaft)
 
 
-#define HISTOGRAM_MEMORY_LIST(HM)                                     \
-  HM(heap_sample_total_committed, V8.MemoryHeapSampleTotalCommitted)  \
-  HM(heap_sample_total_used, V8.MemoryHeapSampleTotalUsed)            \
-  HM(heap_sample_map_space_committed,                                 \
-     V8.MemoryHeapSampleMapSpaceCommitted)                            \
-  HM(heap_sample_code_space_committed,                                \
-     V8.MemoryHeapSampleCodeSpaceCommitted)                           \
-  HM(heap_sample_maximum_committed,                                   \
-     V8.MemoryHeapSampleMaximumCommitted)                             \
+#define HISTOGRAM_LEGACY_MEMORY_LIST(HM)                                      \
+  HM(heap_sample_total_committed, V8.MemoryHeapSampleTotalCommitted)          \
+  HM(heap_sample_total_used, V8.MemoryHeapSampleTotalUsed)                    \
+  HM(heap_sample_map_space_committed, V8.MemoryHeapSampleMapSpaceCommitted)   \
+  HM(heap_sample_code_space_committed, V8.MemoryHeapSampleCodeSpaceCommitted) \
+  HM(heap_sample_maximum_committed, V8.MemoryHeapSampleMaximumCommitted)
+
+#define HISTOGRAM_MEMORY_LIST(HM)                   \
+  HM(memory_heap_committed, V8.MemoryHeapCommitted) \
+  HM(memory_heap_used, V8.MemoryHeapUsed)
 
 
 // WARNING: STATS_COUNTER_LIST_* is a very large macro that is causing MSVC
@@ -620,6 +739,14 @@ class Counters {
 
 #define HM(name, caption) \
   Histogram* name() { return &name##_; }
+  HISTOGRAM_LEGACY_MEMORY_LIST(HM)
+  HISTOGRAM_MEMORY_LIST(HM)
+#undef HM
+
+#define HM(name, caption)                                     \
+  AggregatedMemoryHistogram<Histogram>* aggregated_##name() { \
+    return &aggregated_##name##_;                             \
+  }
   HISTOGRAM_MEMORY_LIST(HM)
 #undef HM
 
@@ -670,6 +797,7 @@ class Counters {
     HISTOGRAM_PERCENTAGE_LIST(PERCENTAGE_ID)
 #undef PERCENTAGE_ID
 #define MEMORY_ID(name, caption) k_##name,
+    HISTOGRAM_LEGACY_MEMORY_LIST(MEMORY_ID)
     HISTOGRAM_MEMORY_LIST(MEMORY_ID)
 #undef MEMORY_ID
 #define COUNTER_ID(name, caption) k_##name,
@@ -718,6 +846,12 @@ class Counters {
 
 #define HM(name, caption) \
   Histogram name##_;
+  HISTOGRAM_LEGACY_MEMORY_LIST(HM)
+  HISTOGRAM_MEMORY_LIST(HM)
+#undef HM
+
+#define HM(name, caption) \
+  AggregatedMemoryHistogram<Histogram> aggregated_##name##_;
   HISTOGRAM_MEMORY_LIST(HM)
 #undef HM
 

src/flag-definitions.h

@@ -648,6 +648,10 @@ DEFINE_IMPLICATION(trace_detached_contexts, track_detached_contexts)
 DEFINE_BOOL(verify_heap, false, "verify heap pointers before and after GC")
 #endif
 
+// counters.cc
+DEFINE_INT(histogram_interval, 600000,
+           "time interval in ms for aggregating memory histograms")
+
 
 // heap-snapshot-generator.cc
 DEFINE_BOOL(heap_profiler_trace_objects, false,

src/heap/gc-tracer.cc

@@ -113,7 +113,7 @@ void GCTracer::Start(GarbageCollector collector, const char* gc_reason,
   if (start_counter_ != 1) return;
 
   previous_ = current_;
-  double start_time = base::OS::TimeCurrentMillis();
+  double start_time = heap_->MonotonicallyIncreasingTimeInMs();
   if (new_space_top_after_gc_ != 0) {
     AddNewSpaceAllocationTime(
         start_time - previous_.end_time,
@@ -154,6 +154,12 @@ void GCTracer::Start(GarbageCollector collector, const char* gc_reason,
   for (int i = 0; i < Scope::NUMBER_OF_SCOPES; i++) {
     current_.scopes[i] = 0;
   }
+  int committed_memory = static_cast<int>(heap_->CommittedMemory() / KB);
+  int used_memory = static_cast<int>(current_.start_object_size / KB);
+  heap_->isolate()->counters()->aggregated_memory_heap_committed()->AddSample(
+      start_time, committed_memory);
+  heap_->isolate()->counters()->aggregated_memory_heap_used()->AddSample(
+      start_time, used_memory);
 }
 
 
@@ -174,13 +180,20 @@ void GCTracer::Stop(GarbageCollector collector) {
           (current_.type == Event::MARK_COMPACTOR ||
            current_.type == Event::INCREMENTAL_MARK_COMPACTOR)));
 
-  current_.end_time = base::OS::TimeCurrentMillis();
+  current_.end_time = heap_->MonotonicallyIncreasingTimeInMs();
   current_.end_object_size = heap_->SizeOfObjects();
   current_.end_memory_size = heap_->isolate()->memory_allocator()->Size();
   current_.end_holes_size = CountTotalHolesSize(heap_);
   new_space_top_after_gc_ =
       reinterpret_cast<intptr_t>(heap_->new_space()->top());
 
+  int committed_memory = static_cast<int>(heap_->CommittedMemory() / KB);
+  int used_memory = static_cast<int>(current_.end_object_size / KB);
+  heap_->isolate()->counters()->aggregated_memory_heap_committed()->AddSample(
+      current_.end_time, committed_memory);
+  heap_->isolate()->counters()->aggregated_memory_heap_used()->AddSample(
+      current_.end_time, used_memory);
+
   if (current_.type == Event::SCAVENGER) {
     current_.incremental_marking_steps =
         current_.cumulative_incremental_marking_steps -

src/heap/heap.cc

@@ -4580,7 +4580,7 @@ bool Heap::TryFinalizeIdleIncrementalMarking(
 }
 
 
-static double MonotonicallyIncreasingTimeInMs() {
+double Heap::MonotonicallyIncreasingTimeInMs() {
   return V8::GetCurrentPlatform()->MonotonicallyIncreasingTime() *
          static_cast<double>(base::Time::kMillisecondsPerSecond);
 }
@@ -4601,7 +4601,8 @@ bool Heap::IdleNotification(double deadline_in_seconds) {
       static_cast<double>(base::Time::kMillisecondsPerSecond);
   HistogramTimerScope idle_notification_scope(
       isolate_->counters()->gc_idle_notification());
-  double idle_time_in_ms = deadline_in_ms - MonotonicallyIncreasingTimeInMs();
+  double start_ms = MonotonicallyIncreasingTimeInMs();
+  double idle_time_in_ms = deadline_in_ms - start_ms;
   bool is_long_idle_notification =
       static_cast<size_t>(idle_time_in_ms) >
       GCIdleTimeHandler::kMaxFrameRenderingIdleTime;
@@ -4645,6 +4646,12 @@ bool Heap::IdleNotification(double deadline_in_seconds) {
       gc_idle_time_handler_.Compute(idle_time_in_ms, heap_state);
   isolate()->counters()->gc_idle_time_allotted_in_ms()->AddSample(
       static_cast<int>(idle_time_in_ms));
+  int committed_memory = static_cast<int>(CommittedMemory() / KB);
+  int used_memory = static_cast<int>(heap_state.size_of_objects / KB);
+  isolate()->counters()->aggregated_memory_heap_committed()->AddSample(
+      start_ms, committed_memory);
+  isolate()->counters()->aggregated_memory_heap_used()->AddSample(start_ms,
+                                                                  used_memory);
 
   bool result = false;
   switch (action.type) {

src/heap/heap.h

@@ -1151,6 +1151,8 @@ class Heap {
   bool IdleNotification(double deadline_in_seconds);
   bool IdleNotification(int idle_time_in_ms);
 
+  double MonotonicallyIncreasingTimeInMs();
+
   // Declare all the root indices.  This defines the root list order.
   enum RootListIndex {
 #define ROOT_INDEX_DECLARATION(type, name, camel_name) k##camel_name##RootIndex,

test/unittests/counters-unittest.cc

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <vector>
+
+#include "src/counters.h"
+#include "src/handles-inl.h"
+#include "src/objects-inl.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+class MockHistogram : public Histogram {
+ public:
+  void AddSample(int value) { samples_.push_back(value); }
+  std::vector<int>* samples() { return &samples_; }
+
+ private:
+  std::vector<int> samples_;
+};
+
+
+class AggregatedMemoryHistogramTest : public ::testing::Test {
+ public:
+  AggregatedMemoryHistogramTest() {
+    aggregated_ = AggregatedMemoryHistogram<MockHistogram>(&mock_);
+  }
+  virtual ~AggregatedMemoryHistogramTest() {}
+
+  void AddSample(double current_ms, double current_value) {
+    aggregated_.AddSample(current_ms, current_value);
+  }
+
+  std::vector<int>* samples() { return mock_.samples(); }
+
+ private:
+  AggregatedMemoryHistogram<MockHistogram> aggregated_;
+  MockHistogram mock_;
+};
+
+}  // namespace
+
+
+TEST_F(AggregatedMemoryHistogramTest, OneSample1) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(20, 1000);
+  EXPECT_EQ(1, samples()->size());
+  EXPECT_EQ(1000, (*samples())[0]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, OneSample2) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 500);
+  AddSample(20, 1000);
+  EXPECT_EQ(1, samples()->size());
+  EXPECT_EQ(750, (*samples())[0]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, OneSample3) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 500);
+  AddSample(15, 500);
+  AddSample(15, 1000);
+  AddSample(20, 1000);
+  EXPECT_EQ(1, samples()->size());
+  EXPECT_EQ(750, (*samples())[0]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, OneSample4) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 500);
+  AddSample(15, 750);
+  AddSample(20, 1000);
+  EXPECT_EQ(1, samples()->size());
+  EXPECT_EQ(750, (*samples())[0]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples1) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(30, 1000);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ(1000, (*samples())[0]);
+  EXPECT_EQ(1000, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples2) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(20, 1000);
+  AddSample(30, 1000);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ(1000, (*samples())[0]);
+  EXPECT_EQ(1000, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples3) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(20, 1000);
+  AddSample(20, 500);
+  AddSample(30, 500);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ(1000, (*samples())[0]);
+  EXPECT_EQ(500, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples4) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(30, 0);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ(750, (*samples())[0]);
+  EXPECT_EQ(250, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples5) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 0);
+  AddSample(30, 1000);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ(250, (*samples())[0]);
+  EXPECT_EQ(750, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples6) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 0);
+  AddSample(15, 1000);
+  AddSample(30, 1000);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ((500 + 1000) / 2, (*samples())[0]);
+  EXPECT_EQ(1000, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples7) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 0);
+  AddSample(15, 1000);
+  AddSample(25, 0);
+  AddSample(30, 1000);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ((500 + 750) / 2, (*samples())[0]);
+  EXPECT_EQ((250 + 500) / 2, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, TwoSamples8) {
+  FLAG_histogram_interval = 10;
+  AddSample(10, 1000);
+  AddSample(15, 0);
+  AddSample(25, 1000);
+  AddSample(30, 0);
+  EXPECT_EQ(2, samples()->size());
+  EXPECT_EQ((500 + 250) / 2, (*samples())[0]);
+  EXPECT_EQ((750 + 500) / 2, (*samples())[1]);
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, ManySamples1) {
+  FLAG_histogram_interval = 10;
+  const int kMaxSamples = 1000;
+  AddSample(0, 0);
+  AddSample(10 * kMaxSamples, 10 * kMaxSamples);
+  EXPECT_EQ(kMaxSamples, samples()->size());
+  for (int i = 0; i < kMaxSamples; i++) {
+    EXPECT_EQ(i * 10 + 5, (*samples())[i]);
+  }
+}
+
+
+TEST_F(AggregatedMemoryHistogramTest, ManySamples2) {
+  FLAG_histogram_interval = 10;
+  const int kMaxSamples = 1000;
+  AddSample(0, 0);
+  AddSample(10 * (2 * kMaxSamples), 10 * (2 * kMaxSamples));
+  EXPECT_EQ(kMaxSamples, samples()->size());
+  for (int i = 0; i < kMaxSamples; i++) {
+    EXPECT_EQ(i * 10 + 5, (*samples())[i]);
+  }
+}
+
+
+}  // namespace internal
+}  // namespace v8

test/unittests/unittests.gyp

@@ -83,6 +83,7 @@
         'compiler/typer-unittest.cc',
         'compiler/value-numbering-reducer-unittest.cc',
         'compiler/zone-pool-unittest.cc',
+        'counters-unittest.cc',
         'libplatform/default-platform-unittest.cc',
         'libplatform/task-queue-unittest.cc',
         'libplatform/worker-thread-unittest.cc',