[turbofan] Preserve NaN properly for NumberMin and NumberMax.

When one of the inputs to NumberMin or NumberMax is NaN we need to
return NaN, ignoring whatever else was passed. Specifically we cannot
lower `NumberMin(x,y)` to `(x < y) ? x : y` if `x` can be NaN. So
limit this optimization to only perform the above lowering if we
know that `x` is an OrderedNumber and `y` is a PlainNumber (or if
the difference between zeros doesn't matter, an OrderedNumber as
well).

Bug: chromium:905457
Change-Id: If05f19255e14789ab0e277e072469c40e161b85b
Reviewed-on: https://chromium-review.googlesource.com/c/1337576Reviewed-by: Georg Neis <neis@chromium.org>
Commit-Queue: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#57535}

src/compiler/simplified-lowering.cc

@@ -2294,9 +2294,13 @@ class RepresentationSelector {
                      UseInfo::TruncatingFloat64(truncation.identify_zeros()),
                      MachineRepresentation::kFloat64);
           if (lower()) {
-            if (truncation.IdentifiesZeroAndMinusZero() ||
-                (lhs_type.Is(Type::PlainNumber()) &&
-                 rhs_type.Is(Type::PlainNumber()))) {
+            // If the right hand side is not NaN, and the left hand side
+            // is not NaN (or -0 if the difference between the zeros is
+            // observed), we can do a simple floating point comparison here.
+            if (lhs_type.Is(truncation.IdentifiesZeroAndMinusZero()
+                                ? Type::OrderedNumber()
+                                : Type::PlainNumber()) &&
+                rhs_type.Is(Type::OrderedNumber())) {
               lowering->DoMax(node, lowering->machine()->Float64LessThan(),
                               MachineRepresentation::kFloat64);
             } else {
@@ -2348,9 +2352,13 @@ class RepresentationSelector {
                      UseInfo::TruncatingFloat64(truncation.identify_zeros()),
                      MachineRepresentation::kFloat64);
           if (lower()) {
-            if (truncation.IdentifiesZeroAndMinusZero() ||
-                (lhs_type.Is(Type::PlainNumber()) &&
-                 rhs_type.Is(Type::PlainNumber()))) {
+            // If the left hand side is not NaN, and the right hand side
+            // is not NaN (or -0 if the difference between the zeros is
+            // observed), we can do a simple floating point comparison here.
+            if (lhs_type.Is(Type::OrderedNumber()) &&
+                rhs_type.Is(truncation.IdentifiesZeroAndMinusZero()
+                                ? Type::OrderedNumber()
+                                : Type::PlainNumber())) {
               lowering->DoMin(node, lowering->machine()->Float64LessThan(),
                               MachineRepresentation::kFloat64);
             } else {

test/mjsunit/regress/regress-crbug-905457.js

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax
+
+(function() {
+  function foo(x) {
+    return Math.abs(Math.min(+x, 0));
+  }
+
+  assertEquals(NaN, foo());
+  assertEquals(NaN, foo());
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(NaN, foo());
+})();
+
+(function() {
+  function foo(x) {
+    return Math.abs(Math.min(-x, 0));
+  }
+
+  assertEquals(NaN, foo());
+  assertEquals(NaN, foo());
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(NaN, foo());
+})();
+
+(function() {
+  function foo(x) {
+    return Math.abs(Math.max(0, +x));
+  }
+
+  assertEquals(NaN, foo());
+  assertEquals(NaN, foo());
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(NaN, foo());
+})();
+
+(function() {
+  function foo(x) {
+    return Math.abs(Math.max(0, -x));
+  }
+
+  assertEquals(NaN, foo());
+  assertEquals(NaN, foo());
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(NaN, foo());
+})();