[scanner] Add a stream for windows-1252

Add a Utf16CharacterStream for windows-1252, which uses a table to
covert from windows-1252 to the appropriate uc16 character. This fixes
issues where windows-1252 encoded source was treated like latin-1.

This introduces a new encoding type in the API, so embedders will have
to opt-in to using this new stream.

Fixed: v8:9315
Change-Id: I2e847c2d4f1727281f6be24b4743d77bb232a62d
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2729338
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Auto-Submit: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/master@{#73134}

include/v8.h

@@ -1936,7 +1936,7 @@ class V8_EXPORT ScriptCompiler {
    */
   class V8_EXPORT StreamedSource {
    public:
-    enum Encoding { ONE_BYTE, TWO_BYTE, UTF8 };
+    enum Encoding { ONE_BYTE, TWO_BYTE, UTF8, WINDOWS_1252 };
 
     V8_DEPRECATED(
         "This class takes ownership of source_stream, so use the constructor "

src/parsing/scanner-character-streams.cc

@@ -411,6 +411,104 @@ bool BufferedUtf16CharacterStream::ReadBlock() {
   return buffer_cursor_ < buffer_end_;
 }
 
+// ----------------------------------------------------------------------------
+// Windows1252CharacterStream - chunked streaming of windows-1252 data.
+//
+// Similar to BufferedCharacterStream, but does the translation of
+// windows-1252 that are incompatible with their latin-1 equivalents.
+
+namespace {
+
+static const uc16 kWindows1252ToUC16[256] = {
+    0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,  // 00-07
+    0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F,  // 08-0F
+    0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,  // 10-17
+    0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F,  // 18-1F
+    0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,  // 20-27
+    0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,  // 28-2F
+    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,  // 30-37
+    0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,  // 38-3F
+    0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,  // 40-47
+    0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,  // 48-4F
+    0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,  // 50-57
+    0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F,  // 58-5F
+    0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,  // 60-67
+    0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,  // 68-6F
+    0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,  // 70-77
+    0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F,  // 78-7F
+    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,  // 80-87
+    0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,  // 88-8F
+    0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,  // 90-97
+    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,  // 98-9F
+    0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,  // A0-A7
+    0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF,  // A8-AF
+    0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7,  // B0-B7
+    0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF,  // B8-BF
+    0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7,  // C0-C7
+    0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,  // C8-CF
+    0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7,  // D0-D7
+    0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF,  // D8-DF
+    0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,  // E0-E7
+    0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,  // E8-EF
+    0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,  // F0-F7
+    0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF   // F8-FF
+};
+
+}  // namespace
+
+class Windows1252CharacterStream final : public Utf16CharacterStream {
+ public:
+  Windows1252CharacterStream(
+      size_t pos, ScriptCompiler::ExternalSourceStream* source_stream)
+      : byte_stream_(source_stream) {
+    buffer_pos_ = pos;
+  }
+
+  bool can_be_cloned() const final {
+    return ChunkedStream<uint16_t>::kCanBeCloned;
+  }
+
+  std::unique_ptr<Utf16CharacterStream> Clone() const override {
+    CHECK(can_be_cloned());
+    return std::unique_ptr<Utf16CharacterStream>(
+        new Windows1252CharacterStream(*this));
+  }
+
+ protected:
+  bool ReadBlock() final {
+    size_t position = pos();
+    buffer_pos_ = position;
+    buffer_start_ = &buffer_[0];
+    buffer_cursor_ = buffer_start_;
+
+    DisallowGarbageCollection no_gc;
+    Range<uint8_t> range =
+        byte_stream_.GetDataAt(position, runtime_call_stats(), &no_gc);
+    if (range.length() == 0) {
+      buffer_end_ = buffer_start_;
+      return false;
+    }
+
+    size_t length = std::min({kBufferSize, range.length()});
+    std::transform(range.start, range.start + length, &buffer_[0],
+                   [](uint8_t c) { return kWindows1252ToUC16[c]; });
+    buffer_end_ = &buffer_[length];
+    return true;
+  }
+
+  bool can_access_heap() const final {
+    return ChunkedStream<uint8_t>::kCanAccessHeap;
+  }
+
+ private:
+  Windows1252CharacterStream(const Windows1252CharacterStream& other)
+      V8_NOEXCEPT : byte_stream_(other.byte_stream_) {}
+
+  static const size_t kBufferSize = 512;
+  uc16 buffer_[kBufferSize];
+  ChunkedStream<uint8_t> byte_stream_;
+};
+
 // ----------------------------------------------------------------------------
 // Utf8ExternalStreamingStream - chunked streaming of Utf-8 data.
 //
@@ -833,6 +931,9 @@ Utf16CharacterStream* ScannerStream::For(
     case v8::ScriptCompiler::StreamedSource::ONE_BYTE:
       return new BufferedCharacterStream<ChunkedStream>(static_cast<size_t>(0),
                                                         source_stream);
+    case v8::ScriptCompiler::StreamedSource::WINDOWS_1252:
+      return new Windows1252CharacterStream(static_cast<size_t>(0),
+                                            source_stream);
     case v8::ScriptCompiler::StreamedSource::UTF8:
       return new Utf8ExternalStreamingStream(source_stream);
   }