[turbofan] Effect linearization after representation inference.

This introduces a compiler pass that schedules the graph and re-wires effect chain according to the schedule. It also connects allocating representation changes to the effect chain, and removes the BeginRegion and EndRegion nodes - they should not be needed anymore because all effectful nodes should be already wired-in.

This is an intermediate CL - the next step is to move lowering of the Change*ToTaggedEffect nodes to StateEffectIntroduction so that we do not have to introduce the effectful versions of nodes.

Review URL: https://codereview.chromium.org/1849603002

Cr-Commit-Position: refs/heads/master@{#35565}

BUILD.gn

@@ -812,6 +812,8 @@ source_set("v8_base") {
     "src/compiler/dead-code-elimination.cc",
     "src/compiler/dead-code-elimination.h",
     "src/compiler/diamond.h",
+    "src/compiler/effect-control-linearizer.cc",
+    "src/compiler/effect-control-linearizer.h",
     "src/compiler/escape-analysis-reducer.cc",
     "src/compiler/escape-analysis-reducer.h",
     "src/compiler/escape-analysis.cc",

src/compiler.h

@@ -158,7 +158,8 @@ class CompilationInfo {
     kSplittingEnabled = 1 << 13,
     kDeoptimizationEnabled = 1 << 14,
     kSourcePositionsEnabled = 1 << 15,
-    kBailoutOnUninitialized = 1 << 16,
+    kEffectSchedulingEnabled = 1 << 16,
+    kBailoutOnUninitialized = 1 << 17,
   };
 
   CompilationInfo(ParseInfo* parse_info, Handle<JSFunction> closure);
@@ -277,6 +278,12 @@ class CompilationInfo {
     return GetFlag(kDeoptimizationEnabled);
   }
 
+  void MarkAsEffectSchedulingEnabled() { SetFlag(kEffectSchedulingEnabled); }
+
+  bool is_effect_scheduling_enabled() const {
+    return GetFlag(kEffectSchedulingEnabled);
+  }
+
   void MarkAsSourcePositionsEnabled() { SetFlag(kSourcePositionsEnabled); }
 
   bool is_source_positions_enabled() const {

src/compiler/effect-control-linearizer.cc

+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/compiler/effect-control-linearizer.h"
+
+#include "src/code-factory.h"
+#include "src/compiler/access-builder.h"
+#include "src/compiler/js-graph.h"
+#include "src/compiler/linkage.h"
+#include "src/compiler/node-properties.h"
+#include "src/compiler/node.h"
+#include "src/compiler/schedule.h"
+
+namespace v8 {
+namespace internal {
+namespace compiler {
+
+EffectControlLinearizer::EffectControlLinearizer(JSGraph* js_graph,
+                                                 Schedule* schedule,
+                                                 Zone* temp_zone)
+    : js_graph_(js_graph), schedule_(schedule), temp_zone_(temp_zone) {}
+
+Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
+CommonOperatorBuilder* EffectControlLinearizer::common() const {
+  return js_graph_->common();
+}
+SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
+  return js_graph_->simplified();
+}
+MachineOperatorBuilder* EffectControlLinearizer::machine() const {
+  return js_graph_->machine();
+}
+
+namespace {
+
+struct BlockEffectData {
+  Node* current_effect = nullptr;  // New effect.
+};
+
+// Effect phis that need to be updated after the first pass.
+struct PendingEffectPhi {
+  Node* effect_phi;
+  BasicBlock* block;
+
+  PendingEffectPhi(Node* effect_phi, BasicBlock* block)
+      : effect_phi(effect_phi), block(block) {}
+};
+
+void UpdateEffectPhi(Node* node, BasicBlock* block,
+                     ZoneVector<BlockEffectData>* block_effects) {
+  // Update all inputs to an effect phi with the effects from the given
+  // block->effect map.
+  DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
+  DCHECK_EQ(node->op()->EffectInputCount(), block->PredecessorCount());
+  for (int i = 0; i < node->op()->EffectInputCount(); i++) {
+    Node* input = node->InputAt(i);
+    BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
+    Node* input_effect =
+        (*block_effects)[predecessor->rpo_number()].current_effect;
+    if (input != input_effect) {
+      node->ReplaceInput(i, input_effect);
+    }
+  }
+}
+
+bool HasIncomingBackEdges(BasicBlock* block) {
+  for (BasicBlock* pred : block->predecessors()) {
+    if (pred->rpo_number() >= block->rpo_number()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void RemoveRegionNode(Node* node) {
+  DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
+         IrOpcode::kBeginRegion == node->opcode());
+  // Update the value/context uses to the value input of the finish node and
+  // the effect uses to the effect input.
+  for (Edge edge : node->use_edges()) {
+    DCHECK(!edge.from()->IsDead());
+    if (NodeProperties::IsEffectEdge(edge)) {
+      edge.UpdateTo(NodeProperties::GetEffectInput(node));
+    } else {
+      DCHECK(!NodeProperties::IsControlEdge(edge));
+      DCHECK(!NodeProperties::IsFrameStateEdge(edge));
+      edge.UpdateTo(node->InputAt(0));
+    }
+  }
+  node->Kill();
+}
+
+}  // namespace
+
+void EffectControlLinearizer::Run() {
+  ZoneVector<BlockEffectData> block_effects(temp_zone());
+  ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
+  block_effects.resize(schedule()->RpoBlockCount());
+  NodeVector inputs_buffer(temp_zone());
+
+  for (BasicBlock* block : *(schedule()->rpo_order())) {
+    size_t instr = 0;
+
+    // The control node should be the first.
+    Node* control = block->NodeAt(instr);
+    DCHECK(NodeProperties::IsControl(control));
+    instr++;
+
+    // Iterate over the phis and update the effect phis.
+    Node* effect = nullptr;
+    Node* terminate = nullptr;
+    for (; instr < block->NodeCount(); instr++) {
+      Node* node = block->NodeAt(instr);
+      // Only go through the phis and effect phis.
+      if (node->opcode() == IrOpcode::kEffectPhi) {
+        // There should be at most one effect phi in a block.
+        DCHECK_NULL(effect);
+        // IfException blocks should not have effect phis.
+        DCHECK_NE(IrOpcode::kIfException, control->opcode());
+        effect = node;
+
+        // Make sure we update the inputs to the incoming blocks' effects.
+        if (HasIncomingBackEdges(block)) {
+          // In case of loops, we do not update the effect phi immediately
+          // because the back predecessor has not been handled yet. We just
+          // record the effect phi for later processing.
+          pending_effect_phis.push_back(PendingEffectPhi(node, block));
+        } else {
+          UpdateEffectPhi(node, block, &block_effects);
+        }
+      } else if (node->opcode() == IrOpcode::kPhi) {
+        // Just skip phis.
+      } else if (node->opcode() == IrOpcode::kTerminate) {
+        DCHECK(terminate == nullptr);
+        terminate = node;
+      } else {
+        break;
+      }
+    }
+
+    if (effect == nullptr) {
+      // There was no effect phi.
+      DCHECK(!HasIncomingBackEdges(block));
+      if (block == schedule()->start()) {
+        // Start block => effect is start.
+        DCHECK_EQ(graph()->start(), control);
+        effect = graph()->start();
+      } else if (control->opcode() == IrOpcode::kEnd) {
+        // End block is just a dummy, no effect needed.
+        DCHECK_EQ(BasicBlock::kNone, block->control());
+        DCHECK_EQ(1u, block->size());
+        effect = nullptr;
+      } else {
+        // If all the predecessors have the same effect, we can use it
+        // as our current effect.
+        int rpo_number = block->PredecessorAt(0)->rpo_number();
+        effect = block_effects[rpo_number].current_effect;
+        for (size_t i = 1; i < block->PredecessorCount(); i++) {
+          int rpo_number = block->PredecessorAt(i)->rpo_number();
+          if (block_effects[rpo_number].current_effect != effect) {
+            effect = nullptr;
+            break;
+          }
+        }
+        if (effect == nullptr) {
+          DCHECK_NE(IrOpcode::kIfException, control->opcode());
+          // The input blocks do not have the same effect. We have
+          // to create an effect phi node.
+          inputs_buffer.clear();
+          inputs_buffer.resize(block->PredecessorCount(), graph()->start());
+          inputs_buffer.push_back(control);
+          effect = graph()->NewNode(
+              common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
+              static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
+          // Let us update the effect phi node later.
+          pending_effect_phis.push_back(PendingEffectPhi(effect, block));
+        } else if (control->opcode() == IrOpcode::kIfException) {
+          // The IfException is connected into the effect chain, so we need
+          // to process it here.
+          ProcessNode(control, &effect, &control);
+        }
+      }
+    }
+
+    // Fixup the Terminate node.
+    if (terminate != nullptr) {
+      NodeProperties::ReplaceEffectInput(terminate, effect);
+    }
+
+    // Process the ordinary instructions.
+    for (; instr < block->NodeCount(); instr++) {
+      Node* node = block->NodeAt(instr);
+      ProcessNode(node, &effect, &control);
+    }
+
+    switch (block->control()) {
+      case BasicBlock::kGoto:
+      case BasicBlock::kNone:
+        break;
+
+      case BasicBlock::kCall:
+      case BasicBlock::kTailCall:
+      case BasicBlock::kBranch:
+      case BasicBlock::kSwitch:
+      case BasicBlock::kReturn:
+      case BasicBlock::kDeoptimize:
+      case BasicBlock::kThrow:
+        ProcessNode(block->control_input(), &effect, &control);
+        break;
+    }
+
+    // Store the effect for later use.
+    block_effects[block->rpo_number()].current_effect = effect;
+  }
+
+  // Update the incoming edges of the effect phis that could not be processed
+  // during the first pass (because they could have incoming back edges).
+  for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
+    UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
+                    &block_effects);
+  }
+}
+
+void EffectControlLinearizer::ProcessNode(Node* node, Node** effect,
+                                          Node** control) {
+  // If the node needs to be wired into the effect/control chain, do this
+  // here.
+  if (TryWireInStateEffect(node, effect, control)) {
+    return;
+  }
+
+  // Remove the end markers of 'atomic' allocation region because the
+  // region should be wired-in now.
+  if (node->opcode() == IrOpcode::kFinishRegion ||
+      node->opcode() == IrOpcode::kBeginRegion) {
+    // Update the value uses to the value input of the finish node and
+    // the effect uses to the effect input.
+
+    // TODO(jarin) Enable this once we make sure everything with side effects
+    // is marked as effectful.
+    if (false) {
+      return RemoveRegionNode(node);
+    }
+  }
+
+  // If the node takes an effect, replace with the current one.
+  if (node->op()->EffectInputCount() > 0) {
+    DCHECK_EQ(1, node->op()->EffectInputCount());
+    Node* input_effect = NodeProperties::GetEffectInput(node);
+
+    if (input_effect != *effect) {
+      NodeProperties::ReplaceEffectInput(node, *effect);
+    }
+
+    // If the node produces an effect, update our current effect. (However,
+    // ignore new effect chains started with ValueEffect.)
+    if (node->op()->EffectOutputCount() > 0) {
+      DCHECK_EQ(1, node->op()->EffectOutputCount());
+      *effect = node;
+    }
+  } else {
+    // New effect chain is only started with a Start or ValueEffect node.
+    DCHECK(node->op()->EffectOutputCount() == 0 ||
+           node->opcode() == IrOpcode::kStart);
+  }
+}
+
+bool EffectControlLinearizer::TryWireInStateEffect(Node* node, Node** effect,
+                                                   Node** control) {
+  ValueEffectControl state(nullptr, nullptr, nullptr);
+  switch (node->opcode()) {
+    case IrOpcode::kChangeInt32ToTagged:
+      state = LowerChangeInt32ToTagged(node, *effect, *control);
+      break;
+    case IrOpcode::kChangeUint32ToTagged:
+      state = LowerChangeUint32ToTagged(node, *effect, *control);
+      break;
+    case IrOpcode::kChangeFloat64ToTagged:
+      state = LowerChangeFloat64ToTagged(node, *effect, *control);
+      break;
+    default:
+      return false;
+  }
+  NodeProperties::ReplaceUses(node, state.value);
+  *effect = state.effect;
+  *control = state.control;
+  return true;
+}
+
+EffectControlLinearizer::ValueEffectControl
+EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node, Node* effect,
+                                                    Node* control) {
+  Node* value = node->InputAt(0);
+
+  Type* const value_type = NodeProperties::GetType(value);
+  Node* const value32 = graph()->NewNode(
+      machine()->TruncateFloat64ToInt32(TruncationMode::kRoundToZero), value);
+  // TODO(bmeurer): This fast case must be disabled until we kill the asm.js
+  // support in the generic JavaScript pipeline, because LoadBuffer is lying
+  // about its result.
+  // if (value_type->Is(Type::Signed32())) {
+  //   return ChangeInt32ToTagged(value32, control);
+  // }
+  Node* check_same = graph()->NewNode(
+      machine()->Float64Equal(), value,
+      graph()->NewNode(machine()->ChangeInt32ToFloat64(), value32));
+  Node* branch_same = graph()->NewNode(common()->Branch(), check_same, control);
+
+  Node* if_smi = graph()->NewNode(common()->IfTrue(), branch_same);
+  Node* vsmi;
+  Node* if_box = graph()->NewNode(common()->IfFalse(), branch_same);
+
+  // We only need to check for -0 if the {value} can potentially contain -0.
+  if (value_type->Maybe(Type::MinusZero())) {
+    Node* check_zero = graph()->NewNode(machine()->Word32Equal(), value32,
+                                        jsgraph()->Int32Constant(0));
+    Node* branch_zero = graph()->NewNode(common()->Branch(BranchHint::kFalse),
+                                         check_zero, if_smi);
+
+    Node* if_zero = graph()->NewNode(common()->IfTrue(), branch_zero);
+    Node* if_notzero = graph()->NewNode(common()->IfFalse(), branch_zero);
+
+    // In case of 0, we need to check the high bits for the IEEE -0 pattern.
+    Node* check_negative = graph()->NewNode(
+        machine()->Int32LessThan(),
+        graph()->NewNode(machine()->Float64ExtractHighWord32(), value),
+        jsgraph()->Int32Constant(0));
+    Node* branch_negative = graph()->NewNode(
+        common()->Branch(BranchHint::kFalse), check_negative, if_zero);
+
+    Node* if_negative = graph()->NewNode(common()->IfTrue(), branch_negative);
+    Node* if_notnegative =
+        graph()->NewNode(common()->IfFalse(), branch_negative);
+
+    // We need to create a box for negative 0.
+    if_smi = graph()->NewNode(common()->Merge(2), if_notzero, if_notnegative);
+    if_box = graph()->NewNode(common()->Merge(2), if_box, if_negative);
+  }
+
+  // On 64-bit machines we can just wrap the 32-bit integer in a smi, for 32-bit
+  // machines we need to deal with potential overflow and fallback to boxing.
+  if (machine()->Is64() || value_type->Is(Type::SignedSmall())) {
+    vsmi = ChangeInt32ToSmi(value32);
+  } else {
+    Node* smi_tag =
+        graph()->NewNode(machine()->Int32AddWithOverflow(), value32, value32);
+
+    Node* check_ovf = graph()->NewNode(common()->Projection(1), smi_tag);
+    Node* branch_ovf = graph()->NewNode(common()->Branch(BranchHint::kFalse),
+                                        check_ovf, if_smi);
+
+    Node* if_ovf = graph()->NewNode(common()->IfTrue(), branch_ovf);
+    if_box = graph()->NewNode(common()->Merge(2), if_ovf, if_box);
+
+    if_smi = graph()->NewNode(common()->IfFalse(), branch_ovf);
+    vsmi = graph()->NewNode(common()->Projection(0), smi_tag);
+  }
+
+  // Allocate the box for the {value}.
+  ValueEffectControl box = AllocateHeapNumberWithValue(value, effect, if_box);
+
+  control = graph()->NewNode(common()->Merge(2), if_smi, box.control);
+  value = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2),
+                           vsmi, box.value, control);
+  effect =
+      graph()->NewNode(common()->EffectPhi(2), effect, box.effect, control);
+  return ValueEffectControl(value, effect, control);
+}
+
+EffectControlLinearizer::ValueEffectControl
+EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node, Node* effect,
+                                                  Node* control) {
+  Node* value = node->InputAt(0);
+
+  if (machine()->Is64() ||
+      NodeProperties::GetType(value)->Is(Type::SignedSmall())) {
+    return ValueEffectControl(ChangeInt32ToSmi(value), effect, control);
+  }
+
+  Node* add = graph()->NewNode(machine()->Int32AddWithOverflow(), value, value);
+
+  Node* ovf = graph()->NewNode(common()->Projection(1), add);
+  Node* branch =
+      graph()->NewNode(common()->Branch(BranchHint::kFalse), ovf, control);
+
+  Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
+  ValueEffectControl alloc =
+      AllocateHeapNumberWithValue(ChangeInt32ToFloat64(value), effect, if_true);
+
+  Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
+  Node* vfalse = graph()->NewNode(common()->Projection(0), add);
+
+  Node* merge = graph()->NewNode(common()->Merge(2), alloc.control, if_false);
+  Node* phi = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2),
+                               alloc.value, vfalse, merge);
+  Node* ephi =
+      graph()->NewNode(common()->EffectPhi(2), alloc.effect, effect, merge);
+
+  return ValueEffectControl(phi, ephi, merge);
+}
+
+EffectControlLinearizer::ValueEffectControl
+EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node, Node* effect,
+                                                   Node* control) {
+  Node* value = node->InputAt(0);
+
+  if (NodeProperties::GetType(value)->Is(Type::UnsignedSmall())) {
+    return ValueEffectControl(ChangeUint32ToSmi(value), effect, control);
+  }
+
+  Node* check = graph()->NewNode(machine()->Uint32LessThanOrEqual(), value,
+                                 SmiMaxValueConstant());
+  Node* branch =
+      graph()->NewNode(common()->Branch(BranchHint::kTrue), check, control);
+
+  Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
+  Node* vtrue = ChangeUint32ToSmi(value);
+
+  Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
+  ValueEffectControl alloc = AllocateHeapNumberWithValue(
+      ChangeUint32ToFloat64(value), effect, if_false);
+
+  Node* merge = graph()->NewNode(common()->Merge(2), if_true, alloc.control);
+  Node* phi = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2),
+                               vtrue, alloc.value, merge);
+  Node* ephi =
+      graph()->NewNode(common()->EffectPhi(2), effect, alloc.effect, merge);
+
+  return ValueEffectControl(phi, ephi, merge);
+}
+
+EffectControlLinearizer::ValueEffectControl
+EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value, Node* effect,
+                                                     Node* control) {
+  // The AllocateHeapNumberStub does not use the context, so we can safely pass
+  // in Smi zero here.
+  Callable callable = CodeFactory::AllocateHeapNumber(jsgraph()->isolate());
+  Node* target = jsgraph()->HeapConstant(callable.code());
+  Node* context = jsgraph()->NoContextConstant();
+  if (!allocate_heap_number_operator_.is_set()) {
+    CallDescriptor* descriptor = Linkage::GetStubCallDescriptor(
+        jsgraph()->isolate(), jsgraph()->zone(), callable.descriptor(), 0,
+        CallDescriptor::kNoFlags, Operator::kNoThrow);
+    allocate_heap_number_operator_.set(common()->Call(descriptor));
+  }
+  Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
+                                       target, context, effect, control);
+  Node* store = graph()->NewNode(
+      machine()->Store(StoreRepresentation(MachineRepresentation::kFloat64,
+                                           kNoWriteBarrier)),
+      heap_number, HeapNumberValueIndexConstant(), value, heap_number, control);
+  return ValueEffectControl(heap_number, store, control);
+}
+
+Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
+  if (machine()->Is64()) {
+    value = graph()->NewNode(machine()->ChangeInt32ToInt64(), value);
+  }
+  return graph()->NewNode(machine()->WordShl(), value, SmiShiftBitsConstant());
+}
+
+Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
+  if (machine()->Is64()) {
+    value = graph()->NewNode(machine()->ChangeUint32ToUint64(), value);
+  }
+  return graph()->NewNode(machine()->WordShl(), value, SmiShiftBitsConstant());
+}
+
+Node* EffectControlLinearizer::ChangeInt32ToFloat64(Node* value) {
+  return graph()->NewNode(machine()->ChangeInt32ToFloat64(), value);
+}
+
+Node* EffectControlLinearizer::ChangeUint32ToFloat64(Node* value) {
+  return graph()->NewNode(machine()->ChangeUint32ToFloat64(), value);
+}
+
+Node* EffectControlLinearizer::HeapNumberValueIndexConstant() {
+  return jsgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
+}
+
+Node* EffectControlLinearizer::SmiMaxValueConstant() {
+  return jsgraph()->Int32Constant(Smi::kMaxValue);
+}
+
+Node* EffectControlLinearizer::SmiShiftBitsConstant() {
+  return jsgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
+}
+
+}  // namespace compiler
+}  // namespace internal
+}  // namespace v8

src/compiler/effect-control-linearizer.h

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_COMPILER_EFFECT_CONTROL_LINEARIZER_H_
+#define V8_COMPILER_EFFECT_CONTROL_LINEARIZER_H_
+
+#include "src/compiler/common-operator.h"
+#include "src/compiler/node.h"
+#include "src/compiler/simplified-operator.h"
+
+namespace v8 {
+namespace internal {
+
+class Zone;
+
+namespace compiler {
+
+class CommonOperatorBuilder;
+class SimplifiedOperatorBuilder;
+class MachineOperatorBuilder;
+class JSGraph;
+class Graph;
+class Schedule;
+
+class EffectControlLinearizer {
+ public:
+  EffectControlLinearizer(JSGraph* graph, Schedule* schedule, Zone* temp_zone);
+
+  void Run();
+
+ private:
+  void ProcessNode(Node* node, Node** current_effect, Node** control);
+
+  struct ValueEffectControl {
+    Node* value;
+    Node* effect;
+    Node* control;
+    ValueEffectControl(Node* value, Node* effect, Node* control)
+        : value(value), effect(effect), control(control) {}
+  };
+
+  bool TryWireInStateEffect(Node* node, Node** effect, Node** control);
+  ValueEffectControl LowerChangeInt32ToTagged(Node* node, Node* effect,
+                                              Node* control);
+  ValueEffectControl LowerChangeUint32ToTagged(Node* node, Node* effect,
+                                               Node* control);
+  ValueEffectControl LowerChangeFloat64ToTagged(Node* node, Node* effect,
+                                                Node* control);
+  ValueEffectControl AllocateHeapNumberWithValue(Node* node, Node* effect,
+                                                 Node* control);
+
+  Node* ChangeInt32ToSmi(Node* value);
+  Node* ChangeUint32ToSmi(Node* value);
+  Node* ChangeInt32ToFloat64(Node* value);
+  Node* ChangeUint32ToFloat64(Node* value);
+
+  Node* HeapNumberValueIndexConstant();
+  Node* SmiMaxValueConstant();
+  Node* SmiShiftBitsConstant();
+
+  JSGraph* jsgraph() const { return js_graph_; }
+  Graph* graph() const;
+  Schedule* schedule() const { return schedule_; }
+  Zone* temp_zone() const { return temp_zone_; }
+  CommonOperatorBuilder* common() const;
+  SimplifiedOperatorBuilder* simplified() const;
+  MachineOperatorBuilder* machine() const;
+
+  JSGraph* js_graph_;
+  Schedule* schedule_;
+  Zone* temp_zone_;
+  SetOncePointer<const Operator> allocate_heap_number_operator_;
+};
+
+}  // namespace compiler
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_COMPILER_EFFECT_CONTROL_LINEARIZER_H_

src/compiler/graph-reducer.cc

@@ -222,7 +222,11 @@ void GraphReducer::ReplaceWithValue(Node* node, Node* value, Node* effect,
         edge.UpdateTo(dead_);
         Revisit(user);
       } else {
-        UNREACHABLE();
+        DCHECK_NOT_NULL(control);
+        edge.UpdateTo(control);
+        Revisit(user);
+        // TODO(jarin) Check that the node cannot throw (otherwise, it
+        // would have to be connected via IfSuccess/IfException).
       }
     } else if (NodeProperties::IsEffectEdge(edge)) {
       DCHECK_NOT_NULL(effect);

src/compiler/pipeline.cc

@@ -19,6 +19,7 @@
 #include "src/compiler/common-operator-reducer.h"
 #include "src/compiler/control-flow-optimizer.h"
 #include "src/compiler/dead-code-elimination.h"
+#include "src/compiler/effect-control-linearizer.h"
 #include "src/compiler/escape-analysis-reducer.h"
 #include "src/compiler/escape-analysis.h"
 #include "src/compiler/frame-elider.h"
@@ -225,6 +226,7 @@ class PipelineData {
     DCHECK(!schedule_);
     schedule_ = schedule;
   }
+  void reset_schedule() { schedule_ = nullptr; }
 
   Zone* instruction_zone() const { return instruction_zone_; }
   InstructionSequence* sequence() const { return sequence_; }
@@ -503,6 +505,9 @@ PipelineCompilationJob::Status PipelineCompilationJob::CreateGraphImpl() {
   if (!info()->shared_info()->asm_function() || FLAG_turbo_asm_deoptimization) {
     info()->MarkAsDeoptimizationEnabled();
   }
+  if (!info()->shared_info()->asm_function()) {
+    info()->MarkAsEffectSchedulingEnabled();
+  }
 
   if (!info()->shared_info()->HasBytecodeArray()) {
     if (!Compiler::EnsureDeoptimizationSupport(info())) return FAILED;
@@ -799,6 +804,35 @@ struct ControlFlowOptimizationPhase {
   }
 };
 
+struct EffectControlLinearizationPhase {
+  static const char* phase_name() { return "effect linearization"; }
+
+  void Run(PipelineData* data, Zone* temp_zone) {
+    // The scheduler requires the graphs to be trimmed, so trim now.
+    // TODO(jarin) Remove the trimming once the scheduler can handle untrimmed
+    // graphs.
+    GraphTrimmer trimmer(temp_zone, data->graph());
+    NodeVector roots(temp_zone);
+    data->jsgraph()->GetCachedNodes(&roots);
+    trimmer.TrimGraph(roots.begin(), roots.end());
+
+    // Schedule the graph without node splitting so that we can
+    // fix the effect and control flow for nodes with low-level side
+    // effects (such as changing representation to tagged or
+    // 'floating' allocation regions.)
+    Schedule* schedule = Scheduler::ComputeSchedule(temp_zone, data->graph(),
+                                                    Scheduler::kNoFlags);
+    if (FLAG_turbo_verify) ScheduleVerifier::Run(schedule);
+
+    // Post-pass for wiring the control/effects
+    // - connect allocating representation changes into the control&effect
+    //   chains and lower them,
+    // - get rid of the region markers,
+    // - introduce effect phis and rewire effects to get SSA again.
+    EffectControlLinearizer introducer(data->jsgraph(), schedule, temp_zone);
+    introducer.Run();
+  }
+};
 
 struct ChangeLoweringPhase {
   static const char* phase_name() { return "change lowering"; }
@@ -823,6 +857,26 @@ struct ChangeLoweringPhase {
   }
 };
 
+struct ComputeEffectSchedulePhase {
+  static const char* phase_name() { return "effect scheduling"; }
+
+  void Run(PipelineData* data, Zone* temp_zone) {
+    Schedule* schedule = Scheduler::ComputeSchedule(temp_zone, data->graph(),
+                                                    Scheduler::kNoFlags);
+    if (FLAG_turbo_verify) ScheduleVerifier::Run(schedule);
+    data->set_schedule(schedule);
+  }
+};
+
+struct EffectScheduleTrimmingPhase {
+  static const char* phase_name() { return "effect schedule graph trimming"; }
+  void Run(PipelineData* data, Zone* temp_zone) {
+    GraphTrimmer trimmer(temp_zone, data->graph());
+    NodeVector roots(temp_zone);
+    data->jsgraph()->GetCachedNodes(&roots);
+    trimmer.TrimGraph(roots.begin(), roots.end());
+  }
+};
 
 struct EarlyGraphTrimmingPhase {
   static const char* phase_name() { return "early graph trimming"; }
@@ -1251,6 +1305,12 @@ Handle<Code> Pipeline::GenerateCode() {
   Run<SimplifiedLoweringPhase>();
   RunPrintAndVerify("Lowered simplified");
 
+  if (info()->is_effect_scheduling_enabled()) {
+    // TODO(jarin) Run value numbering for the representation changes.
+    Run<EffectControlLinearizationPhase>();
+    RunPrintAndVerify("Effect and control linearized");
+  }
+
   Run<BranchEliminationPhase>();
   RunPrintAndVerify("Branch conditions eliminated");
 

src/compiler/schedule.cc

@@ -199,11 +199,28 @@ void Schedule::AddGoto(BasicBlock* block, BasicBlock* succ) {
   AddSuccessor(block, succ);
 }
 
+#if DEBUG
+namespace {
+
+bool IsPotentiallyThrowingCall(IrOpcode::Value opcode) {
+  switch (opcode) {
+#define BUILD_BLOCK_JS_CASE(Name) case IrOpcode::k##Name:
+    JS_OP_LIST(BUILD_BLOCK_JS_CASE)
+#undef BUILD_BLOCK_JS_CASE
+    case IrOpcode::kCall:
+      return true;
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+#endif  // DEBUG
 
 void Schedule::AddCall(BasicBlock* block, Node* call, BasicBlock* success_block,
                        BasicBlock* exception_block) {
   DCHECK_EQ(BasicBlock::kNone, block->control());
-  DCHECK_EQ(IrOpcode::kCall, call->opcode());
+  DCHECK(IsPotentiallyThrowingCall(call->opcode()));
   block->set_control(BasicBlock::kCall);
   AddSuccessor(block, success_block);
   AddSuccessor(block, exception_block);

src/compiler/scheduler.cc

@@ -324,6 +324,10 @@ class CFGBuilder : public ZoneObject {
       case IrOpcode::kSwitch:
         BuildBlocksForSuccessors(node);
         break;
+#define BUILD_BLOCK_JS_CASE(Name) case IrOpcode::k##Name:
+        JS_OP_LIST(BUILD_BLOCK_JS_CASE)
+// JS opcodes are just like calls => fall through.
+#undef BUILD_BLOCK_JS_CASE
       case IrOpcode::kCall:
         if (NodeProperties::IsExceptionalCall(node)) {
           BuildBlocksForSuccessors(node);
@@ -364,6 +368,10 @@ class CFGBuilder : public ZoneObject {
         scheduler_->UpdatePlacement(node, Scheduler::kFixed);
         ConnectThrow(node);
         break;
+#define CONNECT_BLOCK_JS_CASE(Name) case IrOpcode::k##Name:
+        JS_OP_LIST(CONNECT_BLOCK_JS_CASE)
+// JS opcodes are just like calls => fall through.
+#undef CONNECT_BLOCK_JS_CASE
       case IrOpcode::kCall:
         if (NodeProperties::IsExceptionalCall(node)) {
           scheduler_->UpdatePlacement(node, Scheduler::kFixed);

test/unittests/compiler/effect-control-linearizer-unittest.cc

+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/compiler/effect-control-linearizer.h"
+#include "src/compiler/access-builder.h"
+#include "src/compiler/js-graph.h"
+#include "src/compiler/node-properties.h"
+#include "src/compiler/schedule.h"
+#include "src/compiler/simplified-operator.h"
+#include "test/unittests/compiler/graph-unittest.h"
+#include "test/unittests/compiler/node-test-utils.h"
+#include "test/unittests/test-utils.h"
+#include "testing/gmock/include/gmock/gmock.h"
+
+namespace v8 {
+namespace internal {
+namespace compiler {
+
+class EffectControlLinearizerTest : public TypedGraphTest {
+ public:
+  EffectControlLinearizerTest()
+      : TypedGraphTest(3),
+        machine_(zone()),
+        javascript_(zone()),
+        simplified_(zone()),
+        jsgraph_(isolate(), graph(), common(), &javascript_, &simplified_,
+                 &machine_) {}
+
+  JSGraph* jsgraph() { return &jsgraph_; }
+  SimplifiedOperatorBuilder* simplified() { return &simplified_; }
+
+ private:
+  MachineOperatorBuilder machine_;
+  JSOperatorBuilder javascript_;
+  SimplifiedOperatorBuilder simplified_;
+  JSGraph jsgraph_;
+};
+
+namespace {
+
+BasicBlock* AddBlockToSchedule(Schedule* schedule) {
+  BasicBlock* block = schedule->NewBasicBlock();
+  block->set_rpo_number(static_cast<int32_t>(schedule->rpo_order()->size()));
+  schedule->rpo_order()->push_back(block);
+  return block;
+}
+
+}  // namespace
+
+TEST_F(EffectControlLinearizerTest, SimpleLoad) {
+  Schedule schedule(zone());
+
+  // Create the graph.
+  Node* heap_number = NumberConstant(0.5);
+  Node* load = graph()->NewNode(
+      simplified()->LoadField(AccessBuilder::ForHeapNumberValue()), heap_number,
+      graph()->start(), graph()->start());
+  Node* ret = graph()->NewNode(common()->Return(), load, graph()->start(),
+                               graph()->start());
+
+  // Build the basic block structure.
+  BasicBlock* start = schedule.start();
+  schedule.rpo_order()->push_back(start);
+  start->set_rpo_number(0);
+
+  // Populate the basic blocks with nodes.
+  schedule.AddNode(start, graph()->start());
+  schedule.AddNode(start, heap_number);
+  schedule.AddNode(start, load);
+  schedule.AddReturn(start, ret);
+
+  // Run the state effect introducer.
+  EffectControlLinearizer introducer(jsgraph(), &schedule, zone());
+  introducer.Run();
+
+  EXPECT_THAT(load,
+              IsLoadField(AccessBuilder::ForHeapNumberValue(), heap_number,
+                          graph()->start(), graph()->start()));
+  // The return should have reconnected effect edge to the load.
+  EXPECT_THAT(ret, IsReturn(load, load, graph()->start()));
+}
+
+TEST_F(EffectControlLinearizerTest, DiamondLoad) {
+  Schedule schedule(zone());
+
+  // Create the graph.
+  Node* branch =
+      graph()->NewNode(common()->Branch(), Int32Constant(0), graph()->start());
+
+  Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
+  Node* heap_number = NumberConstant(0.5);
+  Node* vtrue = graph()->NewNode(
+      simplified()->LoadField(AccessBuilder::ForHeapNumberValue()), heap_number,
+      graph()->start(), if_true);
+
+  Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
+  Node* vfalse = Float64Constant(2);
+
+  Node* merge = graph()->NewNode(common()->Merge(2), if_true, if_false);
+  Node* phi = graph()->NewNode(
+      common()->Phi(MachineRepresentation::kFloat64, 2), vtrue, vfalse, merge);
+
+  Node* ret =
+      graph()->NewNode(common()->Return(), phi, graph()->start(), merge);
+
+  // Build the basic block structure.
+  BasicBlock* start = schedule.start();
+  schedule.rpo_order()->push_back(start);
+  start->set_rpo_number(0);
+
+  BasicBlock* tblock = AddBlockToSchedule(&schedule);
+  BasicBlock* fblock = AddBlockToSchedule(&schedule);
+  BasicBlock* mblock = AddBlockToSchedule(&schedule);
+
+  // Populate the basic blocks with nodes.
+  schedule.AddNode(start, graph()->start());
+  schedule.AddBranch(start, branch, tblock, fblock);
+
+  schedule.AddNode(tblock, if_true);
+  schedule.AddNode(tblock, heap_number);
+  schedule.AddNode(tblock, vtrue);
+  schedule.AddGoto(tblock, mblock);
+
+  schedule.AddNode(fblock, if_false);
+  schedule.AddNode(fblock, vfalse);
+  schedule.AddGoto(fblock, mblock);
+
+  schedule.AddNode(mblock, merge);
+  schedule.AddNode(mblock, phi);
+  schedule.AddReturn(mblock, ret);
+
+  // Run the state effect introducer.
+  EffectControlLinearizer introducer(jsgraph(), &schedule, zone());
+  introducer.Run();
+
+  // The effect input to the return should be an effect phi with the
+  // newly introduced effectful change operators.
+  ASSERT_THAT(
+      ret, IsReturn(phi, IsEffectPhi(vtrue, graph()->start(), merge), merge));
+}
+
+TEST_F(EffectControlLinearizerTest, LoopLoad) {
+  Schedule schedule(zone());
+
+  // Create the graph.
+  Node* loop = graph()->NewNode(common()->Loop(1), graph()->start());
+  Node* effect_phi =
+      graph()->NewNode(common()->EffectPhi(1), graph()->start(), loop);
+
+  Node* cond = Int32Constant(0);
+  Node* branch = graph()->NewNode(common()->Branch(), cond, loop);
+
+  Node* if_true = graph()->NewNode(common()->IfTrue(), branch);
+
+  Node* if_false = graph()->NewNode(common()->IfFalse(), branch);
+
+  loop->AppendInput(zone(), if_false);
+  NodeProperties::ChangeOp(loop, common()->Loop(2));
+
+  effect_phi->InsertInput(zone(), 1, effect_phi);
+  NodeProperties::ChangeOp(effect_phi, common()->EffectPhi(2));
+
+  Node* heap_number = NumberConstant(0.5);
+  Node* load = graph()->NewNode(
+      simplified()->LoadField(AccessBuilder::ForHeapNumberValue()), heap_number,
+      graph()->start(), loop);
+
+  Node* ret = graph()->NewNode(common()->Return(), load, effect_phi, if_true);
+
+  // Build the basic block structure.
+  BasicBlock* start = schedule.start();
+  schedule.rpo_order()->push_back(start);
+  start->set_rpo_number(0);
+
+  BasicBlock* lblock = AddBlockToSchedule(&schedule);
+  BasicBlock* fblock = AddBlockToSchedule(&schedule);
+  BasicBlock* rblock = AddBlockToSchedule(&schedule);
+
+  // Populate the basic blocks with nodes.
+  schedule.AddNode(start, graph()->start());
+  schedule.AddGoto(start, lblock);
+
+  schedule.AddNode(lblock, loop);
+  schedule.AddNode(lblock, effect_phi);
+  schedule.AddNode(lblock, heap_number);
+  schedule.AddNode(lblock, load);
+  schedule.AddNode(lblock, cond);
+  schedule.AddBranch(lblock, branch, rblock, fblock);
+
+  schedule.AddNode(fblock, if_false);
+  schedule.AddGoto(fblock, lblock);
+
+  schedule.AddNode(rblock, if_true);
+  schedule.AddReturn(rblock, ret);
+
+  // Run the state effect introducer.
+  EffectControlLinearizer introducer(jsgraph(), &schedule, zone());
+  introducer.Run();
+
+  ASSERT_THAT(ret, IsReturn(load, load, if_true));
+  EXPECT_THAT(load, IsLoadField(AccessBuilder::ForHeapNumberValue(),
+                                heap_number, effect_phi, loop));
+}
+
+}  // namespace compiler
+}  // namespace internal
+}  // namespace v8

test/unittests/unittests.gyp

@@ -50,6 +50,7 @@
         'compiler/control-flow-optimizer-unittest.cc',
         'compiler/dead-code-elimination-unittest.cc',
         'compiler/diamond-unittest.cc',
+        'compiler/effect-control-linearizer-unittest.cc',
         'compiler/escape-analysis-unittest.cc',
         'compiler/graph-reducer-unittest.cc',
         'compiler/graph-reducer-unittest.h',

tools/gyp/v8.gyp

@@ -510,6 +510,8 @@
         '../../src/compiler/dead-code-elimination.cc',
         '../../src/compiler/dead-code-elimination.h',
         '../../src/compiler/diamond.h',
+        '../../src/compiler/effect-control-linearizer.cc',
+        '../../src/compiler/effect-control-linearizer.h',
         '../../src/compiler/escape-analysis.cc',
         '../../src/compiler/escape-analysis.h',
         "../../src/compiler/escape-analysis-reducer.cc",