Skip to content

V
V8
Commit 0dddc6fd authored by Michael Lippautz's avatar Michael Lippautz Committed by Commit Bot
Browse files
Options

[object-stats] Add reader for Chrome's trace file format 

Allows reading GC object stats data from an unpacked Chrome trace file.

No-try: true
Bug: v8:7266
Change-Id: I1b851a5b9c5937bd80ae08129d3daee8b4397500
Reviewed-on: https://chromium-review.googlesource.com/868211Reviewed-by: 's avatarCamillo Bruni <cbruni@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50622}
parent 96f55796
Show whitespace changes
Inline Side-by-side
Showing with 217 additions and 142 deletions
tools/heap-stats/README.md
View file @ 0dddc6fd
...@@ -5,7 +5,9 @@ For example, the tool can be used to visualize how much heap memory is used for ...@@ -5,7 +5,9 @@ For example, the tool can be used to visualize how much heap memory is used for
maintaining internal state versus actually allocated by the user. maintaining internal state versus actually allocated by the user.
The tool consumes log files produced by d8 (or Chromium) by passing The tool consumes log files produced by d8 (or Chromium) by passing
`--trace-gc-object-stats`. `--trace-gc-object-stats` or a trace captured using Chrome's tracing
infrastructure. Chrome trace files need to be unpacked before they can
be used though.
Hosting requires a web server, e.g.: Hosting requires a web server, e.g.:
......
tools/heap-stats/details-selection.html
View file @ 0dddc6fd
...@@ -39,6 +39,7 @@ span { ...@@ -39,6 +39,7 @@ span {
margin-top: 10px; margin-top: 10px;
} }
</style> </style>
<h2>Data selection</h2>
<ul> <ul>
<li> <li>
<label for="isolate-select"> <label for="isolate-select">
......
tools/heap-stats/index.html
View file @ 0dddc6fd
...@@ -42,6 +42,8 @@ let state = Object.create(null); ...@@ -42,6 +42,8 @@ let state = Object.create(null);
function globalDataChanged(e) { function globalDataChanged(e) {
state.data = e.detail; state.data = e.detail;
// Emit one entry with the whole model for debugging purposes.
console.log(state.data);
state.selection = null; state.selection = null;
$('#global-timeline').selection = state.selection; $('#global-timeline').selection = state.selection;
$('#global-timeline').data = state.data; $('#global-timeline').data = state.data;
...@@ -62,10 +64,21 @@ function globalSelectionChangedA(e) { ...@@ -62,10 +64,21 @@ function globalSelectionChangedA(e) {
<body> <body>
<trace-file-reader onchange="globalDataChanged(event)"></trace-file-reader> <trace-file-reader onchange="globalDataChanged(event)"></trace-file-reader>
<h1>V8 Heap Statistics</h1> <h1>V8 Heap Statistics</h1>
<p>Visualize object statistics that have been gathered using</p>
<ul>
<li><code>--trace-gc-object-stats on V8</code></li>
<li>
<a
href="https://www.chromium.org/developers/how-tos/trace-event-profiling-tool">Chrome's
tracing infrastructure</a> collecting data for the category
<code>v8.gc_stats</code>. The trace file needs to be unpacked (e.g. using
<code>gunzip</code>).
</li>
</ul>
<p> <p>
Visualize object stats gathered using <code>--trace-gc-object-stats</code>. Note that the visualizer needs to run on a web server due to HTML imports
Needs to be run on a web server (e.g. <code>python -m requiring <a
SimpleHTTPServer</code>) due to HTML imports requiring CORS. href="https://en.wikipedia.org/wiki/Cross-origin_resource_sharing">CORS</a>.
</p> </p>
<details-selection id="details-selection" onchange="globalSelectionChangedA(event)"></details-selection> <details-selection id="details-selection" onchange="globalSelectionChangedA(event)"></details-selection>
<global-timeline id="global-timeline"></global-timeline> <global-timeline id="global-timeline"></global-timeline>
......
tools/heap-stats/trace-file-reader.js
View file @ 0dddc6fd
...@@ -53,17 +53,9 @@ class TraceFileReader extends HTMLElement { ...@@ -53,17 +53,9 @@ class TraceFileReader extends HTMLElement {
const result = new FileReader(); const result = new FileReader();
result.onload = (e) => { result.onload = (e) => {
let contents = e.target.result.split('\n'); let contents = e.target.result.split('\n');
contents = contents.map(function(line) { const return_data = (e.target.result.includes('V8.GC_Objects_Stats')) ?
try { this.createModelFromChromeTraceFile(contents) :
// Strip away a potentially present adb logcat prefix. this.createModelFromV8TraceFile(contents);
line = line.replace(/^I\/v8\s*\(\d+\):\s+/g, '');
return JSON.parse(line);
} catch (e) {
console.log('unable to parse line: \'' + line + '\'\' (' + e + ')');
}
return null;
});
const return_data = this.createModel(contents);
this.updateLabel('Finished loading \'' + file.name + '\'.'); this.updateLabel('Finished loading \'' + file.name + '\'.');
this.dispatchEvent(new CustomEvent( this.dispatchEvent(new CustomEvent(
'change', {bubbles: true, composed: true, detail: return_data})); 'change', {bubbles: true, composed: true, detail: return_data}));
...@@ -71,14 +63,8 @@ class TraceFileReader extends HTMLElement { ...@@ -71,14 +63,8 @@ class TraceFileReader extends HTMLElement {
result.readAsText(file); result.readAsText(file);
} }
createModel(contents) { createOrUpdateEntryIfNeeded(data, keys, entry) {
// contents is an array of JSON objects that is consolidated into the console.assert(entry.isolate, 'entry should have an isolate');
// application model.
const data = Object.create(null); // Final data container.
const keys = Object.create(null); // Collecting 'keys' per isolate.
let createOrUpdateEntryIfNeeded = entry => {
if (!(entry.isolate in keys)) { if (!(entry.isolate in keys)) {
keys[entry.isolate] = new Set(); keys[entry.isolate] = new Set();
} }
...@@ -97,99 +83,53 @@ class TraceFileReader extends HTMLElement { ...@@ -97,99 +83,53 @@ class TraceFileReader extends HTMLElement {
if (('id' in entry) && !(entry.id in data_object.gcs)) { if (('id' in entry) && !(entry.id in data_object.gcs)) {
data_object.gcs[entry.id] = {non_empty_instance_types: new Set()}; data_object.gcs[entry.id] = {non_empty_instance_types: new Set()};
} }
if (data_object.end === null || data_object.end < entry.time) { if ('time' in entry) {
if (data_object.end === null || data_object.end < entry.time)
data_object.end = entry.time; data_object.end = entry.time;
}
if (data_object.start === null || data_object.start > entry.time) if (data_object.start === null || data_object.start > entry.time)
data_object.start = entry.time; data_object.start = entry.time;
};
for (var entry of contents) {
if (entry === null || entry.type === undefined) {
continue;
} }
if (entry.type === 'zone') { }
createOrUpdateEntryIfNeeded(entry);
const stacktrace = ('stacktrace' in entry) ? entry.stacktrace : []; createDatasetIfNeeded(data, keys, entry, data_set) {
data[entry.isolate].samples.zone[entry.time] = { if (!(data_set in data[entry.isolate].gcs[entry.id])) {
allocated: entry.allocated, data[entry.isolate].gcs[entry.id][data_set] = {
pooled: entry.pooled,
stacktrace: stacktrace
};
} else if (
entry.type === 'zonecreation' || entry.type === 'zonedestruction') {
createOrUpdateEntryIfNeeded(entry);
data[entry.isolate].zonetags.push(
Object.assign({opening: entry.type === 'zonecreation'}, entry));
} else if (entry.type === 'gc_descriptor') {
createOrUpdateEntryIfNeeded(entry);
data[entry.isolate].gcs[entry.id].time = entry.time;
if ('zone' in entry)
data[entry.isolate].gcs[entry.id].malloced = entry.zone;
} else if (entry.type === 'instance_type_data') {
if (entry.id in data[entry.isolate].gcs) {
createOrUpdateEntryIfNeeded(entry);
if (!(entry.key in data[entry.isolate].gcs[entry.id])) {
data[entry.isolate].gcs[entry.id][entry.key] = {
instance_type_data: {}, instance_type_data: {},
non_empty_instance_types: new Set(), non_empty_instance_types: new Set(),
overall: 0 overall: 0
}; };
data[entry.isolate].data_sets.add(entry.key); data[entry.isolate].data_sets.add(data_set);
} }
const instanceTypeName = entry.instance_type_name; }
const id = entry.id;
const key = entry.key; addInstanceTypeData(
keys[entry.isolate].add(key); data, keys, isolate, gc_id, data_set, instance_type, entry) {
data[entry.isolate] keys[isolate].add(data_set);
.gcs[id][key] data[isolate].gcs[gc_id][data_set].instance_type_data[instance_type] = {
.instance_type_data[instanceTypeName] = {
overall: entry.overall, overall: entry.overall,
count: entry.count, count: entry.count,
histogram: entry.histogram, histogram: entry.histogram,
over_allocated: entry.over_allocated, over_allocated: entry.over_allocated,
over_allocated_histogram: entry.over_allocated_histogram over_allocated_histogram: entry.over_allocated_histogram
}; };
data[entry.isolate].gcs[id][key].overall += entry.overall; data[isolate].gcs[gc_id][data_set].overall += entry.overall;
if (entry.overall !== 0) { if (entry.overall !== 0) {
data[entry.isolate].gcs[id][key].non_empty_instance_types.add( data[isolate].gcs[gc_id][data_set].non_empty_instance_types.add(
instanceTypeName); instance_type);
data[entry.isolate].gcs[id].non_empty_instance_types.add( data[isolate].gcs[gc_id].non_empty_instance_types.add(instance_type);
instanceTypeName); data[isolate].non_empty_instance_types.add(instance_type);
data[entry.isolate].non_empty_instance_types.add(instanceTypeName);
}
}
} else if (entry.type === 'bucket_sizes') {
if (entry.id in data[entry.isolate].gcs) {
createOrUpdateEntryIfNeeded(entry);
if (!(entry.key in data[entry.isolate].gcs[entry.id])) {
data[entry.isolate].gcs[entry.id][entry.key] = {
instance_type_data: {},
non_empty_instance_types: new Set(),
overall: 0
};
data[entry.isolate].data_sets.add(entry.key);
}
data[entry.isolate].gcs[entry.id][entry.key].bucket_sizes =
entry.sizes;
}
} else {
console.warning('Unknown entry type: ' + entry.type);
} }
} }
let checkNonNegativeProperty = (obj, property) => { extendAndSanitizeModel(data, keys) {
if (obj[property] < 0) { const checkNonNegativeProperty = (obj, property) => {
console.warning( console.assert(obj[property] >= 0, 'negative property', obj, property);
'Property \'' + property + '\' negative: ' + obj[property]);
}
}; };
for (const isolate of Object.keys(data)) { for (const isolate of Object.keys(data)) {
for (const gc of Object.keys(data[isolate].gcs)) { for (const gc of Object.keys(data[isolate].gcs)) {
for (const key of keys[isolate]) { for (const data_set_key of keys[isolate]) {
const data_set = data[isolate].gcs[gc][key]; const data_set = data[isolate].gcs[gc][data_set_key];
// 1. Create a ranked instance type array that sorts instance // 1. Create a ranked instance type array that sorts instance
// types by memory size (overall). // types by memory size (overall).
data_set.ranked_instance_types = data_set.ranked_instance_types =
...@@ -205,35 +145,154 @@ class TraceFileReader extends HTMLElement { ...@@ -205,35 +145,154 @@ class TraceFileReader extends HTMLElement {
return 0; return 0;
}); });
// 2. Create *FIXED_ARRAY_UNKNOWN_SUB_TYPE that accounts for all let known_count = 0;
// missing fixed array sub types. let known_overall = 0;
const fixed_array_data = let known_histogram =
Object.assign({}, data_set.instance_type_data.FIXED_ARRAY_TYPE); Array(
for (const instanceType in data_set.instance_type_data) { data_set.instance_type_data.FIXED_ARRAY_TYPE.histogram.length)
if (!instanceType.startsWith('*FIXED_ARRAY')) continue; .fill(0);
const subtype = data_set.instance_type_data[instanceType]; for (const instance_type in data_set.instance_type_data) {
fixed_array_data.count -= subtype.count; if (!instance_type.startsWith('*FIXED_ARRAY')) continue;
fixed_array_data.overall -= subtype.overall; const subtype = data_set.instance_type_data[instance_type];
for (let i = 0; i < fixed_array_data.histogram.length; i++) { known_count += subtype.count;
fixed_array_data.histogram[i] -= subtype.histogram[i]; known_overall += subtype.count;
for (let i = 0; i < subtype.histogram.length; i++) {
known_histogram[i] += subtype.histogram[i];
} }
} }
// Emit log messages for negative values. const fixed_array_data = data_set.instance_type_data.FIXED_ARRAY_TYPE;
checkNonNegativeProperty(fixed_array_data, 'count'); const unknown_entry = {
checkNonNegativeProperty(fixed_array_data, 'overall'); count: fixed_array_data.count - known_count,
for (let i = 0; i < fixed_array_data.histogram.length; i++) { overall: fixed_array_data.overall - known_overall,
checkNonNegativeProperty(fixed_array_data.histogram, i); histogram: fixed_array_data.histogram.map(
(value, index) => value - known_histogram[index])
};
// Check for non-negative values.
checkNonNegativeProperty(unknown_entry, 'count');
checkNonNegativeProperty(unknown_entry, 'overall');
for (let i = 0; i < unknown_entry.histogram.length; i++) {
checkNonNegativeProperty(unknown_entry.histogram, i);
} }
data_set.instance_type_data['*FIXED_ARRAY_UNKNOWN_SUB_TYPE'] = data_set.instance_type_data['*FIXED_ARRAY_UNKNOWN_SUB_TYPE'] =
fixed_array_data; unknown_entry;
data_set.non_empty_instance_types.add( data_set.non_empty_instance_types.add(
'*FIXED_ARRAY_UNKNOWN_SUB_TYPE'); '*FIXED_ARRAY_UNKNOWN_SUB_TYPE');
} }
} }
} }
console.log(data); }
createModelFromChromeTraceFile(contents) {
console.log('Processing log as chrome trace file.');
const data = Object.create(null); // Final data container.
const keys = Object.create(null); // Collecting 'keys' per isolate.
// Pop last line in log as it might be broken.
contents.pop();
// Remove trailing comma.
contents[contents.length - 1] = contents[contents.length - 1].slice(0, -1);
// Terminate JSON.
const sanitized_contents = [...contents, ']}'].join('');
try {
const raw_data = JSON.parse(sanitized_contents);
const objects_stats_data =
raw_data.traceEvents.filter(e => e.name == 'V8.GC_Objects_Stats');
objects_stats_data.forEach(trace_data => {
const actual_data = trace_data.args;
const data_sets = new Set(Object.keys(actual_data));
Object.keys(actual_data).forEach(data_set => {
const string_entry = actual_data[data_set];
try {
const entry = JSON.parse(string_entry);
this.createOrUpdateEntryIfNeeded(data, keys, entry);
this.createDatasetIfNeeded(data, keys, entry, data_set);
const isolate = entry.isolate;
const time = entry.time;
const gc_id = entry.id;
data[isolate].gcs[gc_id].time = time;
data[isolate].gcs[gc_id][data_set].bucket_sizes =
entry.bucket_sizes;
for (let [instance_type, value] of Object.entries(
entry.type_data)) {
// Trace file format uses markers that do not have actual
// properties.
if (!('overall' in value)) continue;
this.addInstanceTypeData(
data, keys, isolate, gc_id, data_set, instance_type, value);
}
} catch (e) {
console.log('Unable to parse data set entry', e);
}
});
});
} catch (e) {
console.log('Unable to parse chrome trace file.', e);
}
this.extendAndSanitizeModel(data, keys);
return data;
}
createModelFromV8TraceFile(contents) {
console.log('Processing log as V8 trace file.');
contents = contents.map(function(line) {
try {
// Strip away a potentially present adb logcat prefix.
line = line.replace(/^I\/v8\s*\(\d+\):\s+/g, '');
return JSON.parse(line);
} catch (e) {
console.log('Unable to parse line: \'' + line + '\'\' (' + e + ')');
}
return null;
});
const data = Object.create(null); // Final data container.
const keys = Object.create(null); // Collecting 'keys' per isolate.
for (var entry of contents) {
if (entry === null || entry.type === undefined) {
continue;
}
if (entry.type === 'zone') {
this.createOrUpdateEntryIfNeeded(data, keys, entry);
const stacktrace = ('stacktrace' in entry) ? entry.stacktrace : [];
data[entry.isolate].samples.zone[entry.time] = {
allocated: entry.allocated,
pooled: entry.pooled,
stacktrace: stacktrace
};
} else if (
entry.type === 'zonecreation' || entry.type === 'zonedestruction') {
this.createOrUpdateEntryIfNeeded(data, keys, entry);
data[entry.isolate].zonetags.push(
Object.assign({opening: entry.type === 'zonecreation'}, entry));
} else if (entry.type === 'gc_descriptor') {
this.createOrUpdateEntryIfNeeded(data, keys, entry);
data[entry.isolate].gcs[entry.id].time = entry.time;
if ('zone' in entry)
data[entry.isolate].gcs[entry.id].malloced = entry.zone;
} else if (entry.type === 'instance_type_data') {
if (entry.id in data[entry.isolate].gcs) {
this.createOrUpdateEntryIfNeeded(data, keys, entry);
this.createDatasetIfNeeded(data, keys, entry, entry.key);
this.addInstanceTypeData(
data, keys, entry.isolate, entry.id, entry.key,
entry.instance_type_name, entry);
}
} else if (entry.type === 'bucket_sizes') {
if (entry.id in data[entry.isolate].gcs) {
this.createOrUpdateEntryIfNeeded(data, keys, entry);
this.createDatasetIfNeeded(data, keys, entry, entry.key);
data[entry.isolate].gcs[entry.id][entry.key].bucket_sizes =
entry.sizes;
}
} else {
console.log('Unknown entry type: ' + entry.type);
}
}
this.extendAndSanitizeModel(data, keys);
return data; return data;
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment