int64-lowering.cc 43.6 KB
Newer Older
1 2 3 4 5
// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/int64-lowering.h"
6

7
#include "src/compiler/common-operator.h"
8
#include "src/compiler/diamond.h"
9
#include "src/compiler/graph.h"
10
#include "src/compiler/linkage.h"
11
#include "src/compiler/machine-operator.h"
12
#include "src/compiler/node-matchers.h"
13
#include "src/compiler/node-properties.h"
14
#include "src/compiler/node.h"
15
#include "src/compiler/wasm-compiler.h"
16 17
// TODO(wasm): Remove this include.
#include "src/wasm/wasm-linkage.h"
18
#include "src/zone/zone.h"
19 20 21 22 23

namespace v8 {
namespace internal {
namespace compiler {

24 25 26 27 28
Int64Lowering::Int64Lowering(
    Graph* graph, MachineOperatorBuilder* machine,
    CommonOperatorBuilder* common, Zone* zone,
    Signature<MachineRepresentation>* signature,
    std::unique_ptr<Int64LoweringSpecialCase> special_case)
29 30
    : zone_(zone),
      graph_(graph),
31 32
      machine_(machine),
      common_(common),
33
      state_(graph, 3),
34
      stack_(zone),
35 36
      replacements_(nullptr),
      signature_(signature),
37 38 39
      placeholder_(
          graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
      special_case_(std::move(special_case)) {
40 41
  DCHECK_NOT_NULL(graph);
  DCHECK_NOT_NULL(graph->end());
42
  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
43 44 45
  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
}

46
void Int64Lowering::LowerGraph() {
47
  if (!machine()->Is32()) {
48 49
    return;
  }
50
  stack_.push_back({graph()->end(), 0});
51 52 53
  state_.Set(graph()->end(), State::kOnStack);

  while (!stack_.empty()) {
54
    NodeState& top = stack_.back();
55 56
    if (top.input_index == top.node->InputCount()) {
      // All inputs of top have already been lowered, now lower top.
57
      stack_.pop_back();
58 59
      state_.Set(top.node, State::kVisited);
      LowerNode(top.node);
60
    } else {
61 62 63
      // Push the next input onto the stack.
      Node* input = top.node->InputAt(top.input_index++);
      if (state_.Get(input) == State::kUnvisited) {
64 65 66 67 68
        if (input->opcode() == IrOpcode::kPhi) {
          // To break cycles with phi nodes we push phis on a separate stack so
          // that they are processed after all other nodes.
          PreparePhiReplacement(input);
          stack_.push_front({input, 0});
69 70
        } else if (input->opcode() == IrOpcode::kEffectPhi ||
                   input->opcode() == IrOpcode::kLoop) {
71
          stack_.push_front({input, 0});
72 73 74
        } else {
          stack_.push_back({input, 0});
        }
75
        state_.Set(input, State::kOnStack);
76 77 78 79 80
      }
    }
  }
}

81 82
namespace {

83
int GetReturnIndexAfterLowering(const CallDescriptor* call_descriptor,
84
                                int old_index) {
85 86
  int result = old_index;
  for (int i = 0; i < old_index; i++) {
87
    if (call_descriptor->GetReturnType(i).representation() ==
88 89 90 91 92 93 94
        MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

95
int GetReturnCountAfterLowering(const CallDescriptor* call_descriptor) {
96
  return GetReturnIndexAfterLowering(
97
      call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
98 99 100
}

int GetParameterIndexAfterLowering(
101 102
    Signature<MachineRepresentation>* signature, int old_index) {
  int result = old_index;
103 104 105 106
  // Be robust towards special indexes (>= param count).
  int max_to_check =
      std::min(old_index, static_cast<int>(signature->parameter_count()));
  for (int i = 0; i < max_to_check; i++) {
107 108 109 110 111 112 113
    if (signature->GetParam(i) == MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

114 115 116 117 118 119 120 121 122 123 124 125
int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
  int result = static_cast<int>(signature->return_count());
  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
    if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
      result++;
    }
  }
  return result;
}

}  // namespace

126 127
void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
  DCHECK_EQ(5, node->InputCount());
128
  LowerMemoryBaseAndIndex(node);
129 130 131 132 133 134 135
  Node* value = node->InputAt(2);
  node->ReplaceInput(2, GetReplacementLow(value));
  node->InsertInput(zone(), 3, GetReplacementHigh(value));
  NodeProperties::ChangeOp(node, op);
  ReplaceNodeWithProjections(node);
}

136
void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
137
  DefaultLowering(node, true);
138
  NodeProperties::ChangeOp(node, op);
139
  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
140 141
}

142
// static
143
int Int64Lowering::GetParameterCountAfterLowering(
144
    Signature<MachineRepresentation>* signature) {
145 146
  // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
  // after lowering.
147 148 149 150
  return GetParameterIndexAfterLowering(
      signature, static_cast<int>(signature->parameter_count()));
}

151 152
void Int64Lowering::GetIndexNodes(Node* index, Node** index_low,
                                  Node** index_high) {
153
#if defined(V8_TARGET_LITTLE_ENDIAN)
154 155 156
  *index_low = index;
  *index_high = graph()->NewNode(machine()->Int32Add(), index,
                                 graph()->NewNode(common()->Int32Constant(4)));
157
#elif defined(V8_TARGET_BIG_ENDIAN)
158 159 160
  *index_low = graph()->NewNode(machine()->Int32Add(), index,
                                graph()->NewNode(common()->Int32Constant(4)));
  *index_high = index;
161 162 163
#endif
}

164
void Int64Lowering::LowerNode(Node* node) {
165 166
  switch (node->opcode()) {
    case IrOpcode::kInt64Constant: {
167
      int64_t value = OpParameter<int64_t>(node->op());
168 169 170 171
      Node* low_node = graph()->NewNode(
          common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
      Node* high_node = graph()->NewNode(
          common()->Int32Constant(static_cast<int32_t>(value >> 32)));
172 173 174
      ReplaceNode(node, low_node, high_node);
      break;
    }
175 176 177 178 179 180
    case IrOpcode::kLoad:
    case IrOpcode::kUnalignedLoad: {
      MachineRepresentation rep;
      if (node->opcode() == IrOpcode::kLoad) {
        rep = LoadRepresentationOf(node->op()).representation();
      } else {
181
        DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
182
        rep = LoadRepresentationOf(node->op()).representation();
183
      }
184

185
      if (rep == MachineRepresentation::kWord64) {
186
        LowerMemoryBaseAndIndex(node);
187 188
        Node* base = node->InputAt(0);
        Node* index = node->InputAt(1);
189 190
        Node* index_low;
        Node* index_high;
191
        GetIndexNodes(index, &index_low, &index_high);
192 193 194 195 196
        const Operator* load_op;

        if (node->opcode() == IrOpcode::kLoad) {
          load_op = machine()->Load(MachineType::Int32());
        } else {
197
          DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
198 199 200
          load_op = machine()->UnalignedLoad(MachineType::Int32());
        }

201 202 203 204 205 206 207 208 209 210 211 212
        Node* high_node;
        if (node->InputCount() > 2) {
          Node* effect_high = node->InputAt(2);
          Node* control_high = node->InputAt(3);
          high_node = graph()->NewNode(load_op, base, index_high, effect_high,
                                       control_high);
          // change the effect change from old_node --> old_effect to
          // old_node --> high_node --> old_effect.
          node->ReplaceInput(2, high_node);
        } else {
          high_node = graph()->NewNode(load_op, base, index_high);
        }
213
        node->ReplaceInput(1, index_low);
214 215
        NodeProperties::ChangeOp(node, load_op);
        ReplaceNode(node, node, high_node);
216 217
      } else {
        DefaultLowering(node);
218 219 220
      }
      break;
    }
221 222 223 224 225 226
    case IrOpcode::kStore:
    case IrOpcode::kUnalignedStore: {
      MachineRepresentation rep;
      if (node->opcode() == IrOpcode::kStore) {
        rep = StoreRepresentationOf(node->op()).representation();
      } else {
227
        DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
228 229 230 231
        rep = UnalignedStoreRepresentationOf(node->op());
      }

      if (rep == MachineRepresentation::kWord64) {
232 233 234 235
        // We change the original store node to store the low word, and create
        // a new store node to store the high word. The effect and control edges
        // are copied from the original store to the new store node, the effect
        // edge of the original store is redirected to the new store.
236
        LowerMemoryBaseAndIndex(node);
237 238
        Node* base = node->InputAt(0);
        Node* index = node->InputAt(1);
239 240
        Node* index_low;
        Node* index_high;
241
        GetIndexNodes(index, &index_low, &index_high);
242 243 244 245
        Node* value = node->InputAt(2);
        DCHECK(HasReplacementLow(value));
        DCHECK(HasReplacementHigh(value));

246 247 248 249 250 251 252
        const Operator* store_op;
        if (node->opcode() == IrOpcode::kStore) {
          WriteBarrierKind write_barrier_kind =
              StoreRepresentationOf(node->op()).write_barrier_kind();
          store_op = machine()->Store(StoreRepresentation(
              MachineRepresentation::kWord32, write_barrier_kind));
        } else {
253
          DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
254 255
          store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
        }
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270

        Node* high_node;
        if (node->InputCount() > 3) {
          Node* effect_high = node->InputAt(3);
          Node* control_high = node->InputAt(4);
          high_node = graph()->NewNode(store_op, base, index_high,
                                       GetReplacementHigh(value), effect_high,
                                       control_high);
          node->ReplaceInput(3, high_node);

        } else {
          high_node = graph()->NewNode(store_op, base, index_high,
                                       GetReplacementHigh(value));
        }

271
        node->ReplaceInput(1, index_low);
272 273 274
        node->ReplaceInput(2, GetReplacementLow(value));
        NodeProperties::ChangeOp(node, store_op);
        ReplaceNode(node, node, high_node);
275
      } else {
276
        DefaultLowering(node, true);
277
      }
278 279
      break;
    }
280 281 282
    case IrOpcode::kStart: {
      int parameter_count = GetParameterCountAfterLowering(signature());
      // Only exchange the node if the parameter count actually changed.
283
      if (parameter_count != static_cast<int>(signature()->parameter_count())) {
284 285 286 287 288 289 290 291
        int delta =
            parameter_count - static_cast<int>(signature()->parameter_count());
        int new_output_count = node->op()->ValueOutputCount() + delta;
        NodeProperties::ChangeOp(node, common()->Start(new_output_count));
      }
      break;
    }
    case IrOpcode::kParameter: {
292
      DCHECK_EQ(1, node->InputCount());
293
      int param_count = static_cast<int>(signature()->parameter_count());
294 295 296 297
      // Only exchange the node if the parameter count actually changed. We do
      // not even have to do the default lowering because the the start node,
      // the only input of a parameter node, only changes if the parameter count
      // changes.
298
      if (GetParameterCountAfterLowering(signature()) != param_count) {
299
        int old_index = ParameterIndexOf(node->op());
300 301
        // Adjust old_index to be compliant with the signature.
        --old_index;
302
        int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303
        // Adjust new_index to consider the instance parameter.
304
        ++new_index;
305
        NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306

307 308 309 310 311 312
        if (old_index < 0 || old_index >= param_count) {
          // Special parameters (JS closure/context) don't have kWord64
          // representation anyway.
          break;
        }

313 314
        if (signature()->GetParam(old_index) ==
            MachineRepresentation::kWord64) {
315 316 317
          Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
                                             graph()->start());
          ReplaceNode(node, node, high_node);
318
        }
319 320 321 322
      }
      break;
    }
    case IrOpcode::kReturn: {
323
      int input_count = node->InputCount();
324
      DefaultLowering(node);
325 326 327 328 329
      if (input_count != node->InputCount()) {
        int new_return_count = GetReturnCountAfterLowering(signature());
        if (static_cast<int>(signature()->return_count()) != new_return_count) {
          NodeProperties::ChangeOp(node, common()->Return(new_return_count));
        }
330
      }
331 332
      break;
    }
333
    case IrOpcode::kTailCall: {
334
      auto call_descriptor =
335
          const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
336
      bool returns_require_lowering =
337 338
          GetReturnCountAfterLowering(call_descriptor) !=
          static_cast<int>(call_descriptor->ReturnCount());
339
      if (DefaultLowering(node) || returns_require_lowering) {
340 341
        // Tail calls do not have return values, so adjusting the call
        // descriptor is enough.
342 343
        NodeProperties::ChangeOp(
            node, common()->TailCall(LowerCallDescriptor(call_descriptor)));
344 345 346
      }
      break;
    }
347
    case IrOpcode::kCall: {
348 349
      auto call_descriptor = CallDescriptorOf(node->op());

350
      bool returns_require_lowering =
351 352
          GetReturnCountAfterLowering(call_descriptor) !=
          static_cast<int>(call_descriptor->ReturnCount());
353
      if (DefaultLowering(node) || returns_require_lowering) {
354
        // We have to adjust the call descriptor.
355 356
        NodeProperties::ChangeOp(
            node, common()->Call(LowerCallDescriptor(call_descriptor)));
357
      }
358
      if (returns_require_lowering) {
359
        size_t return_arity = call_descriptor->ReturnCount();
360 361
        if (return_arity == 1) {
          // We access the additional return values through projections.
362
          ReplaceNodeWithProjections(node);
363 364 365 366 367 368 369 370
        } else {
          ZoneVector<Node*> projections(return_arity, zone());
          NodeProperties::CollectValueProjections(node, projections.data(),
                                                  return_arity);
          for (size_t old_index = 0, new_index = 0; old_index < return_arity;
               ++old_index, ++new_index) {
            Node* use_node = projections[old_index];
            DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
371
            DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
372 373 374 375 376 377
                                                  static_cast<int>(old_index)),
                      static_cast<int>(new_index));
            if (new_index != old_index) {
              NodeProperties::ChangeOp(
                  use_node, common()->Projection(new_index));
            }
378
            if (call_descriptor->GetReturnType(old_index).representation() ==
379 380 381 382 383 384 385 386 387
                MachineRepresentation::kWord64) {
              Node* high_node = graph()->NewNode(
                  common()->Projection(new_index + 1), node,
                  graph()->start());
              ReplaceNode(use_node, use_node, high_node);
              ++new_index;
            }
          }
        }
388 389
      }
      break;
390
    }
391
    case IrOpcode::kWord64And: {
392
      DCHECK_EQ(2, node->InputCount());
393 394 395 396 397 398 399 400 401 402 403 404 405
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
    case IrOpcode::kTruncateInt64ToInt32: {
406
      DCHECK_EQ(1, node->InputCount());
407 408 409 410 411
      Node* input = node->InputAt(0);
      ReplaceNode(node, GetReplacementLow(input), nullptr);
      node->NullAllInputs();
      break;
    }
412
    case IrOpcode::kInt64Add: {
413
      DCHECK_EQ(2, node->InputCount());
414 415 416 417 418 419 420 421 422 423 424

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
      // We access the additional return values through projections.
425
      ReplaceNodeWithProjections(node);
426 427
      break;
    }
428
    case IrOpcode::kInt64Sub: {
429
      DCHECK_EQ(2, node->InputCount());
430 431 432 433 434 435 436 437 438 439 440

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairSub());
      // We access the additional return values through projections.
441
      ReplaceNodeWithProjections(node);
442 443
      break;
    }
444
    case IrOpcode::kInt64Mul: {
445
      DCHECK_EQ(2, node->InputCount());
446 447 448 449 450 451 452 453 454 455 456

      Node* right = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(right));
      node->AppendInput(zone(), GetReplacementHigh(right));

      Node* left = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(left));
      node->InsertInput(zone(), 1, GetReplacementHigh(left));

      NodeProperties::ChangeOp(node, machine()->Int32PairMul());
      // We access the additional return values through projections.
457
      ReplaceNodeWithProjections(node);
458 459
      break;
    }
460
    case IrOpcode::kWord64Or: {
461
      DCHECK_EQ(2, node->InputCount());
462 463 464 465 466 467 468 469 470 471 472 473
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
474
    case IrOpcode::kWord64Xor: {
475
      DCHECK_EQ(2, node->InputCount());
476 477 478 479 480 481 482 483 484 485 486 487
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      Node* low_node =
          graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
                           GetReplacementLow(right));
      Node* high_node =
          graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
                           GetReplacementHigh(right));
      ReplaceNode(node, low_node, high_node);
      break;
    }
488 489 490
    case IrOpcode::kWord64Shl: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
491
      DCHECK_EQ(2, node->InputCount());
492 493 494 495 496 497 498 499 500 501 502 503 504
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairShl());
      // We access the additional return values through projections.
505
      ReplaceNodeWithProjections(node);
506 507
      break;
    }
508 509 510
    case IrOpcode::kWord64Shr: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
511
      DCHECK_EQ(2, node->InputCount());
512 513 514 515 516 517 518 519 520 521 522 523 524
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairShr());
      // We access the additional return values through projections.
525
      ReplaceNodeWithProjections(node);
526 527 528 529 530
      break;
    }
    case IrOpcode::kWord64Sar: {
      // TODO(turbofan): if the shift count >= 32, then we can set the low word
      // of the output to 0 and just calculate the high word.
531
      DCHECK_EQ(2, node->InputCount());
532 533 534 535 536 537 538 539 540 541 542 543 544
      Node* shift = node->InputAt(1);
      if (HasReplacementLow(shift)) {
        // We do not have to care about the high word replacement, because
        // the shift can only be between 0 and 63 anyways.
        node->ReplaceInput(1, GetReplacementLow(shift));
      }

      Node* value = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(value));
      node->InsertInput(zone(), 1, GetReplacementHigh(value));

      NodeProperties::ChangeOp(node, machine()->Word32PairSar());
      // We access the additional return values through projections.
545
      ReplaceNodeWithProjections(node);
546 547
      break;
    }
548
    case IrOpcode::kWord64Equal: {
549
      DCHECK_EQ(2, node->InputCount());
550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
      Node* left = node->InputAt(0);
      Node* right = node->InputAt(1);

      // TODO(wasm): Use explicit comparisons and && here?
      Node* replacement = graph()->NewNode(
          machine()->Word32Equal(),
          graph()->NewNode(
              machine()->Word32Or(),
              graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
                               GetReplacementLow(right)),
              graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
                               GetReplacementHigh(right))),
          graph()->NewNode(common()->Int32Constant(0)));

      ReplaceNode(node, replacement, nullptr);
      break;
    }
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
    case IrOpcode::kInt64LessThan: {
      LowerComparison(node, machine()->Int32LessThan(),
                      machine()->Uint32LessThan());
      break;
    }
    case IrOpcode::kInt64LessThanOrEqual: {
      LowerComparison(node, machine()->Int32LessThan(),
                      machine()->Uint32LessThanOrEqual());
      break;
    }
    case IrOpcode::kUint64LessThan: {
      LowerComparison(node, machine()->Uint32LessThan(),
                      machine()->Uint32LessThan());
      break;
    }
    case IrOpcode::kUint64LessThanOrEqual: {
      LowerComparison(node, machine()->Uint32LessThan(),
                      machine()->Uint32LessThanOrEqual());
      break;
    }
587
    case IrOpcode::kSignExtendWord32ToInt64:
588
    case IrOpcode::kChangeInt32ToInt64: {
589
      DCHECK_EQ(1, node->InputCount());
590 591 592 593 594 595 596 597 598 599 600 601 602
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kChangeUint32ToUint64: {
603
      DCHECK_EQ(1, node->InputCount());
604 605 606 607 608 609 610 611
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
      node->NullAllInputs();
      break;
    }
612
    case IrOpcode::kBitcastInt64ToFloat64: {
613
      DCHECK_EQ(1, node->InputCount());
614 615 616 617 618 619 620 621
      Node* input = node->InputAt(0);
      Node* stack_slot = graph()->NewNode(
          machine()->StackSlot(MachineRepresentation::kWord64));

      Node* store_high_word = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kWord32,
                                  WriteBarrierKind::kNoWriteBarrier)),
622
          stack_slot,
623 624
          graph()->NewNode(
              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
625 626 627 628 629 630
          GetReplacementHigh(input), graph()->start(), graph()->start());

      Node* store_low_word = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kWord32,
                                  WriteBarrierKind::kNoWriteBarrier)),
631
          stack_slot,
632 633
          graph()->NewNode(
              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
634 635 636 637 638 639 640 641 642 643
          GetReplacementLow(input), store_high_word, graph()->start());

      Node* load =
          graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
                           graph()->NewNode(common()->Int32Constant(0)),
                           store_low_word, graph()->start());

      ReplaceNode(node, load, nullptr);
      break;
    }
644
    case IrOpcode::kBitcastFloat64ToInt64: {
645
      DCHECK_EQ(1, node->InputCount());
646
      Node* input = node->InputAt(0);
647 648 649
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
650 651 652 653 654 655 656 657 658
      Node* stack_slot = graph()->NewNode(
          machine()->StackSlot(MachineRepresentation::kWord64));
      Node* store = graph()->NewNode(
          machine()->Store(
              StoreRepresentation(MachineRepresentation::kFloat64,
                                  WriteBarrierKind::kNoWriteBarrier)),
          stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
          graph()->start(), graph()->start());

659 660
      Node* high_node = graph()->NewNode(
          machine()->Load(MachineType::Int32()), stack_slot,
661 662 663
          graph()->NewNode(
              common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
          store, graph()->start());
664

665 666
      Node* low_node = graph()->NewNode(
          machine()->Load(MachineType::Int32()), stack_slot,
667 668 669
          graph()->NewNode(
              common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
          store, graph()->start());
670 671 672
      ReplaceNode(node, low_node, high_node);
      break;
    }
673 674 675
    case IrOpcode::kWord64Rol:
      DCHECK(machine()->Word32Rol().IsSupported());
      V8_FALLTHROUGH;
676
    case IrOpcode::kWord64Ror: {
677
      DCHECK_EQ(2, node->InputCount());
678 679 680 681 682
      Node* input = node->InputAt(0);
      Node* shift = HasReplacementLow(node->InputAt(1))
                        ? GetReplacementLow(node->InputAt(1))
                        : node->InputAt(1);
      Int32Matcher m(shift);
683
      if (m.HasResolvedValue()) {
684
        // Precondition: 0 <= shift < 64.
685
        int32_t shift_value = m.ResolvedValue() & 0x3F;
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
        if (shift_value == 0) {
          ReplaceNode(node, GetReplacementLow(input),
                      GetReplacementHigh(input));
        } else if (shift_value == 32) {
          ReplaceNode(node, GetReplacementHigh(input),
                      GetReplacementLow(input));
        } else {
          Node* low_input;
          Node* high_input;
          if (shift_value < 32) {
            low_input = GetReplacementLow(input);
            high_input = GetReplacementHigh(input);
          } else {
            low_input = GetReplacementHigh(input);
            high_input = GetReplacementLow(input);
          }
702
          int32_t masked_shift_value = shift_value & 0x1F;
703 704 705 706 707
          Node* masked_shift =
              graph()->NewNode(common()->Int32Constant(masked_shift_value));
          Node* inv_shift = graph()->NewNode(
              common()->Int32Constant(32 - masked_shift_value));

708 709 710 711 712 713 714 715 716 717 718 719 720
          auto* op1 = machine()->Word32Shr();
          auto* op2 = machine()->Word32Shl();
          bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
          if (!is_ror) std::swap(op1, op2);

          Node* low_node =
              graph()->NewNode(machine()->Word32Or(),
                               graph()->NewNode(op1, low_input, masked_shift),
                               graph()->NewNode(op2, high_input, inv_shift));
          Node* high_node =
              graph()->NewNode(machine()->Word32Or(),
                               graph()->NewNode(op1, high_input, masked_shift),
                               graph()->NewNode(op2, low_input, inv_shift));
721 722 723 724 725 726 727
          ReplaceNode(node, low_node, high_node);
        }
      } else {
        Node* safe_shift = shift;
        if (!machine()->Word32ShiftIsSafe()) {
          safe_shift =
              graph()->NewNode(machine()->Word32And(), shift,
728
                               graph()->NewNode(common()->Int32Constant(0x1F)));
729 730
        }

731 732 733 734 735 736 737
        bool is_ror = node->opcode() == IrOpcode::kWord64Ror;
        Node* inv_mask =
            is_ror ? graph()->NewNode(
                         machine()->Word32Xor(),
                         graph()->NewNode(
                             machine()->Word32Shr(),
                             graph()->NewNode(common()->Int32Constant(-1)),
738
                             safe_shift),
739 740 741 742 743
                         graph()->NewNode(common()->Int32Constant(-1)))
                   : graph()->NewNode(
                         machine()->Word32Shl(),
                         graph()->NewNode(common()->Int32Constant(-1)),
                         safe_shift);
744 745 746 747 748 749 750 751 752 753 754 755

        Node* bit_mask =
            graph()->NewNode(machine()->Word32Xor(), inv_mask,
                             graph()->NewNode(common()->Int32Constant(-1)));

        // We have to mask the shift value for this comparison. If
        // !machine()->Word32ShiftIsSafe() then the masking should already be
        // part of the graph.
        Node* masked_shift6 = shift;
        if (machine()->Word32ShiftIsSafe()) {
          masked_shift6 =
              graph()->NewNode(machine()->Word32And(), shift,
756
                               graph()->NewNode(common()->Int32Constant(0x3F)));
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
        }

        Diamond lt32(
            graph(), common(),
            graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
                             graph()->NewNode(common()->Int32Constant(32))));

        // The low word and the high word can be swapped either at the input or
        // at the output. We swap the inputs so that shift does not have to be
        // kept for so long in a register.
        Node* input_low =
            lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
                     GetReplacementHigh(input));
        Node* input_high =
            lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
                     GetReplacementLow(input));

774 775 776 777 778 779 780 781 782
        const Operator* oper =
            is_ror ? machine()->Word32Ror() : machine()->Word32Rol().op();

        Node* rotate_low = graph()->NewNode(oper, input_low, safe_shift);
        Node* rotate_high = graph()->NewNode(oper, input_high, safe_shift);

        auto* mask1 = bit_mask;
        auto* mask2 = inv_mask;
        if (!is_ror) std::swap(mask1, mask2);
783 784 785

        Node* low_node = graph()->NewNode(
            machine()->Word32Or(),
786 787
            graph()->NewNode(machine()->Word32And(), rotate_low, mask1),
            graph()->NewNode(machine()->Word32And(), rotate_high, mask2));
788 789
        Node* high_node = graph()->NewNode(
            machine()->Word32Or(),
790 791
            graph()->NewNode(machine()->Word32And(), rotate_high, mask1),
            graph()->NewNode(machine()->Word32And(), rotate_low, mask2));
792 793 794 795
        ReplaceNode(node, low_node, high_node);
      }
      break;
    }
796
    case IrOpcode::kWord64Clz: {
797
      DCHECK_EQ(1, node->InputCount());
798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
      Node* input = node->InputAt(0);
      Diamond d(
          graph(), common(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
                           graph()->NewNode(common()->Int32Constant(0))));

      Node* low_node = d.Phi(
          MachineRepresentation::kWord32,
          graph()->NewNode(machine()->Int32Add(),
                           graph()->NewNode(machine()->Word32Clz(),
                                            GetReplacementLow(input)),
                           graph()->NewNode(common()->Int32Constant(32))),
          graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
814
    case IrOpcode::kWord64Ctz: {
815
      DCHECK_EQ(1, node->InputCount());
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
      DCHECK(machine()->Word32Ctz().IsSupported());
      Node* input = node->InputAt(0);
      Diamond d(
          graph(), common(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
                           graph()->NewNode(common()->Int32Constant(0))));
      Node* low_node =
          d.Phi(MachineRepresentation::kWord32,
                graph()->NewNode(machine()->Int32Add(),
                                 graph()->NewNode(machine()->Word32Ctz().op(),
                                                  GetReplacementHigh(input)),
                                 graph()->NewNode(common()->Int32Constant(32))),
                graph()->NewNode(machine()->Word32Ctz().op(),
                                 GetReplacementLow(input)));
      ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
833
    case IrOpcode::kWord64Popcnt: {
834
      DCHECK_EQ(1, node->InputCount());
835 836 837 838 839 840 841 842 843 844 845 846 847
      Node* input = node->InputAt(0);
      // We assume that a Word64Popcnt node only has been created if
      // Word32Popcnt is actually supported.
      DCHECK(machine()->Word32Popcnt().IsSupported());
      ReplaceNode(node, graph()->NewNode(
                            machine()->Int32Add(),
                            graph()->NewNode(machine()->Word32Popcnt().op(),
                                             GetReplacementLow(input)),
                            graph()->NewNode(machine()->Word32Popcnt().op(),
                                             GetReplacementHigh(input))),
                  graph()->NewNode(common()->Int32Constant(0)));
      break;
    }
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863
    case IrOpcode::kPhi: {
      MachineRepresentation rep = PhiRepresentationOf(node->op());
      if (rep == MachineRepresentation::kWord64) {
        // The replacement nodes have already been created, we only have to
        // replace placeholder nodes.
        Node* low_node = GetReplacementLow(node);
        Node* high_node = GetReplacementHigh(node);
        for (int i = 0; i < node->op()->ValueInputCount(); i++) {
          low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
          high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
        }
      } else {
        DefaultLowering(node);
      }
      break;
    }
864 865
    case IrOpcode::kWord64ReverseBytes: {
      Node* input = node->InputAt(0);
866 867 868 869
      ReplaceNode(node,
                  graph()->NewNode(machine()->Word32ReverseBytes(),
                                   GetReplacementHigh(input)),
                  graph()->NewNode(machine()->Word32ReverseBytes(),
870 871 872
                                   GetReplacementLow(input)));
      break;
    }
873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
    case IrOpcode::kSignExtendWord8ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // Sign extend low node to Int32
      input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);

      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
    case IrOpcode::kSignExtendWord16ToInt64: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      if (HasReplacementLow(input)) {
        input = GetReplacementLow(input);
      }
      // Sign extend low node to Int32
      input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);

      // We use SAR to preserve the sign in the high word.
      ReplaceNode(
          node, input,
          graph()->NewNode(machine()->Word32Sar(), input,
                           graph()->NewNode(common()->Int32Constant(31))));
      node->NullAllInputs();
      break;
    }
907 908 909
    case IrOpcode::kWord64AtomicLoad: {
      DCHECK_EQ(4, node->InputCount());
      MachineType type = AtomicOpType(node->op());
910
      DefaultLowering(node, true);
911 912 913 914 915 916 917 918 919 920 921 922 923
      if (type == MachineType::Uint64()) {
        NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
        ReplaceNodeWithProjections(node);
      } else {
        NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
      }
      break;
    }
    case IrOpcode::kWord64AtomicStore: {
      DCHECK_EQ(5, node->InputCount());
      MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
      if (rep == MachineRepresentation::kWord64) {
924
        LowerMemoryBaseAndIndex(node);
925 926 927 928 929 930 931 932 933 934
        Node* value = node->InputAt(2);
        node->ReplaceInput(2, GetReplacementLow(value));
        node->InsertInput(zone(), 3, GetReplacementHigh(value));
        NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
      } else {
        DefaultLowering(node, true);
        NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
      }
      break;
    }
935 936 937 938 939 940
#define ATOMIC_CASE(name)                                                   \
  case IrOpcode::kWord64Atomic##name: {                                     \
    MachineType type = AtomicOpType(node->op());                            \
    if (type == MachineType::Uint64()) {                                    \
      LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name());    \
    } else {                                                                \
941
      LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
942 943
    }                                                                       \
    break;                                                                  \
944 945 946 947 948 949
  }
      ATOMIC_CASE(Add)
      ATOMIC_CASE(Sub)
      ATOMIC_CASE(And)
      ATOMIC_CASE(Or)
      ATOMIC_CASE(Xor)
950
      ATOMIC_CASE(Exchange)
951
#undef ATOMIC_CASE
952 953 954
    case IrOpcode::kWord64AtomicCompareExchange: {
      MachineType type = AtomicOpType(node->op());
      if (type == MachineType::Uint64()) {
955
        LowerMemoryBaseAndIndex(node);
956 957 958 959 960 961 962 963 964 965
        Node* old_value = node->InputAt(2);
        Node* new_value = node->InputAt(3);
        node->ReplaceInput(2, GetReplacementLow(old_value));
        node->ReplaceInput(3, GetReplacementHigh(old_value));
        node->InsertInput(zone(), 4, GetReplacementLow(new_value));
        node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
        NodeProperties::ChangeOp(node,
                                 machine()->Word32AtomicPairCompareExchange());
        ReplaceNodeWithProjections(node);
      } else {
966 967
        DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
               type == MachineType::Uint8());
968
        DefaultLowering(node, true);
969 970 971
        NodeProperties::ChangeOp(node,
                                 machine()->Word32AtomicCompareExchange(type));
        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
972 973 974
      }
      break;
    }
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000
    case IrOpcode::kI64x2Splat: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      node->ReplaceInput(0, GetReplacementLow(input));
      node->AppendInput(zone(), GetReplacementHigh(input));
      NodeProperties::ChangeOp(node, machine()->I64x2SplatI32Pair());
      break;
    }
    case IrOpcode::kI64x2ExtractLane: {
      DCHECK_EQ(1, node->InputCount());
      Node* input = node->InputAt(0);
      int32_t lane = OpParameter<int32_t>(node->op());
      ReplaceNode(
          node, graph()->NewNode(machine()->I32x4ExtractLane(lane * 2), input),
          graph()->NewNode(machine()->I32x4ExtractLane(lane * 2 + 1), input));
      break;
    }
    case IrOpcode::kI64x2ReplaceLane: {
      DCHECK_EQ(2, node->InputCount());
      int32_t lane = OpParameter<int32_t>(node->op());
      Node* input = node->InputAt(1);
      node->ReplaceInput(1, GetReplacementLow(input));
      node->AppendInput(zone(), GetReplacementHigh(input));
      NodeProperties::ChangeOp(node, machine()->I64x2ReplaceLaneI32Pair(lane));
      break;
    }
1001

1002
    default: { DefaultLowering(node); }
1003
  }
1004
}  // NOLINT(readability/fn_size)
1005

1006 1007
void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
                                    const Operator* low_word_op) {
1008
  DCHECK_EQ(2, node->InputCount());
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);
  Node* replacement = graph()->NewNode(
      machine()->Word32Or(),
      graph()->NewNode(high_word_op, GetReplacementHigh(left),
                       GetReplacementHigh(right)),
      graph()->NewNode(
          machine()->Word32And(),
          graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
                           GetReplacementHigh(right)),
          graph()->NewNode(low_word_op, GetReplacementLow(left),
                           GetReplacementLow(right))));

  ReplaceNode(node, replacement, nullptr);
}

1025
bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
1026 1027 1028 1029 1030 1031 1032
  bool something_changed = false;
  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
    Node* input = node->InputAt(i);
    if (HasReplacementLow(input)) {
      something_changed = true;
      node->ReplaceInput(i, GetReplacementLow(input));
    }
1033
    if (!low_word_only && HasReplacementHigh(input)) {
1034 1035 1036 1037 1038 1039 1040
      something_changed = true;
      node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
    }
  }
  return something_changed;
}

1041
const CallDescriptor* Int64Lowering::LowerCallDescriptor(
1042 1043
    const CallDescriptor* call_descriptor) {
  if (special_case_) {
1044 1045 1046
    auto replacement = special_case_->replacements.find(call_descriptor);
    if (replacement != special_case_->replacements.end()) {
      return replacement->second;
1047 1048 1049 1050 1051
    }
  }
  return GetI32WasmCallDescriptor(zone(), call_descriptor);
}

1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
  // if new_low == nullptr, then also new_high == nullptr.
  DCHECK(new_low != nullptr || new_high == nullptr);
  replacements_[old->id()].low = new_low;
  replacements_[old->id()].high = new_high;
}

bool Int64Lowering::HasReplacementLow(Node* node) {
  return replacements_[node->id()].low != nullptr;
}

Node* Int64Lowering::GetReplacementLow(Node* node) {
  Node* result = replacements_[node->id()].low;
  DCHECK(result);
  return result;
}

bool Int64Lowering::HasReplacementHigh(Node* node) {
  return replacements_[node->id()].high != nullptr;
}

Node* Int64Lowering::GetReplacementHigh(Node* node) {
  Node* result = replacements_[node->id()].high;
  DCHECK(result);
  return result;
}
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103

void Int64Lowering::PreparePhiReplacement(Node* phi) {
  MachineRepresentation rep = PhiRepresentationOf(phi->op());
  if (rep == MachineRepresentation::kWord64) {
    // We have to create the replacements for a phi node before we actually
    // lower the phi to break potential cycles in the graph. The replacements of
    // input nodes do not exist yet, so we use a placeholder node to pass the
    // graph verifier.
    int value_count = phi->op()->ValueInputCount();
    Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
    Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
    for (int i = 0; i < value_count; i++) {
      inputs_low[i] = placeholder_;
      inputs_high[i] = placeholder_;
    }
    inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
    inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
    ReplaceNode(phi,
                graph()->NewNode(
                    common()->Phi(MachineRepresentation::kWord32, value_count),
                    value_count + 1, inputs_low, false),
                graph()->NewNode(
                    common()->Phi(MachineRepresentation::kWord32, value_count),
                    value_count + 1, inputs_high, false));
  }
}
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113

void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
  DCHECK(node != nullptr);
  Node* low_node =
      graph()->NewNode(common()->Projection(0), node, graph()->start());
  Node* high_node =
      graph()->NewNode(common()->Projection(1), node, graph()->start());
  ReplaceNode(node, low_node, high_node);
}

1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
  DCHECK(node != nullptr);
  // Low word only replacements for memory operands for 32-bit address space.
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  if (HasReplacementLow(base)) {
    node->ReplaceInput(0, GetReplacementLow(base));
  }
  if (HasReplacementLow(index)) {
    node->ReplaceInput(1, GetReplacementLow(index));
  }
}

1127 1128 1129
}  // namespace compiler
}  // namespace internal
}  // namespace v8