gcmole.lua 14.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
-- Copyright 2011 the V8 project authors. All rights reserved.
-- Redistribution and use in source and binary forms, with or without
-- modification, are permitted provided that the following conditions are
-- met:
--
--     * Redistributions of source code must retain the above copyright
--       notice, this list of conditions and the following disclaimer.
--     * Redistributions in binary form must reproduce the above
--       copyright notice, this list of conditions and the following
--       disclaimer in the documentation and/or other materials provided
--       with the distribution.
--     * Neither the name of Google Inc. nor the names of its
--       contributors may be used to endorse or promote products derived
--       from this software without specific prior written permission.
--
-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

-- This is main driver for gcmole tool. See README for more details.
-- Usage: CLANG_BIN=clang-bin-dir lua tools/gcmole/gcmole.lua [arm|ia32|x64]

local DIR = arg[0]:match("^(.+)/[^/]+$")
32 33 34 35 36

local FLAGS = {
   -- Do not build gcsuspects file and reuse previously generated one.
   reuse_gcsuspects = false;

37 38 39
   -- Don't use parallel python runner.
   sequential = false;

40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
   -- Print commands to console before executing them.
   verbose = false;

   -- Perform dead variable analysis (generates many false positives).
   -- TODO add some sort of whiteliste to filter out false positives.
   dead_vars = false;

   -- When building gcsuspects whitelist certain functions as if they
   -- can be causing GC. Currently used to reduce number of false
   -- positives in dead variables analysis. See TODO for WHITELIST
   -- below.
   whitelist = true;
}
local ARGS = {}

for i = 1, #arg do
   local flag = arg[i]:match "^%-%-([%w_-]+)$"
   if flag then
      local no, real_flag = flag:match "^(no)([%w_-]+)$"
      if real_flag then flag = real_flag end

      flag = flag:gsub("%-", "_")
      if FLAGS[flag] ~= nil then
         FLAGS[flag] = (no ~= "no")
      else
         error("Unknown flag: " .. flag)
      end
   else
      table.insert(ARGS, arg[i])
   end
end

72
local ARCHS = ARGS[1] and { ARGS[1] } or { 'ia32', 'arm', 'x64', 'arm64' }
73 74 75 76 77 78 79 80 81 82 83 84

local io = require "io"
local os = require "os"

function log(...)
   io.stderr:write(string.format(...))
   io.stderr:write "\n"
end

-------------------------------------------------------------------------------
-- Clang invocation

85
local CLANG_BIN = os.getenv "CLANG_BIN"
86
local CLANG_PLUGINS = os.getenv "CLANG_PLUGINS"
87 88 89

if not CLANG_BIN or CLANG_BIN == "" then
   error "CLANG_BIN not set"
90
end
91

92 93 94 95
if not CLANG_PLUGINS or CLANG_PLUGINS == "" then
   CLANG_PLUGINS = DIR
end

96 97
local function MakeClangCommandLine(
      plugin, plugin_args, triple, arch_define, arch_options)
98 99
   if plugin_args then
     for i = 1, #plugin_args do
100 101
        plugin_args[i] = "-Xclang -plugin-arg-" .. plugin
           .. " -Xclang " .. plugin_args[i]
102 103 104
     end
     plugin_args = " " .. table.concat(plugin_args, " ")
   end
105
   return CLANG_BIN .. "/clang++ -std=c++14 -c"
106 107
      .. " -Xclang -load -Xclang " .. CLANG_PLUGINS .. "/libgcmole.so"
      .. " -Xclang -plugin -Xclang "  .. plugin
108
      .. (plugin_args or "")
109
      .. " -Xclang -triple -Xclang " .. triple
110
      .. " -fno-exceptions"
111 112
      .. " -D" .. arch_define
      .. " -DENABLE_DEBUGGER_SUPPORT"
113
      .. " -DV8_INTL_SUPPORT"
114
      .. " -I./"
115
      .. " -Iinclude/"
116
      .. " -Iout/Release/gen"
117 118
      .. " -Ithird_party/icu/source/common"
      .. " -Ithird_party/icu/source/i18n"
119
      .. " " .. arch_options
120 121
end

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
local function IterTable(t)
  return coroutine.wrap(function ()
    for i, v in ipairs(t) do
      coroutine.yield(v)
    end
  end)
end

local function SplitResults(lines, func)
   -- Splits the output of parallel.py and calls func on each result.
   -- Bails out in case of an error in one of the executions.
   local current = {}
   local filename = ""
   for line in lines do
      local new_file = line:match "^______________ (.*)$"
      local code = line:match "^______________ finish (%d+) ______________$"
      if code then
         if tonumber(code) > 0 then
            log(table.concat(current, "\n"))
            log("Failed to examine " .. filename)
            return false
         end
         log("-- %s", filename)
         func(filename, IterTable(current))
      elseif new_file then
         filename = new_file
         current = {}
      else
         table.insert(current, line)
      end
   end
   return true
end

156 157
function InvokeClangPluginForEachFile(filenames, cfg, func)
   local cmd_line = MakeClangCommandLine(cfg.plugin,
158 159
                                         cfg.plugin_args,
                                         cfg.triple,
160 161
                                         cfg.arch_define,
                                         cfg.arch_options)
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
   if FLAGS.sequential then
      log("** Sequential execution.")
      for _, filename in ipairs(filenames) do
         log("-- %s", filename)
         local action = cmd_line .. " " .. filename .. " 2>&1"
         if FLAGS.verbose then print('popen ', action) end
         local pipe = io.popen(action)
         func(filename, pipe:lines())
         local success = pipe:close()
         if not success then error("Failed to run: " .. action) end
      end
   else
      log("** Parallel execution.")
      local action = "python tools/gcmole/parallel.py \""
         .. cmd_line .. "\" " .. table.concat(filenames, " ")
177
      if FLAGS.verbose then print('popen ', action) end
178
      local pipe = io.popen(action)
179 180 181
      local success = SplitResults(pipe:lines(), func)
      local closed = pipe:close()
      if not (success and closed) then error("Failed to run: " .. action) end
182 183 184 185 186
   end
end

-------------------------------------------------------------------------------

187
local function ParseGNFile(for_test)
188
   local result = {}
189 190 191 192 193 194 195 196 197 198 199
   local gn_files
   if for_test then
      gn_files = {
         { "tools/gcmole/GCMOLE.gn",             '"([^"]-%.cc)"',      ""         }
      }
   else
      gn_files = {
         { "BUILD.gn",             '"([^"]-%.cc)"',      ""         },
         { "test/cctest/BUILD.gn", '"(test-[^"]-%.cc)"', "test/cctest/" }
      }
   end
200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219

   for i = 1, #gn_files do
      local filename = gn_files[i][1]
      local pattern = gn_files[i][2]
      local prefix = gn_files[i][3]
      local gn_file = assert(io.open(filename), "failed to open GN file")
      local gn = gn_file:read('*a')
      for condition, sources in
         gn:gmatch "### gcmole%((.-)%) ###(.-)%]" do
         if result[condition] == nil then result[condition] = {} end
         for file in sources:gmatch(pattern) do
            table.insert(result[condition], prefix .. file)
         end
      end
      gn_file:close()
   end

   return result
end

220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
local function EvaluateCondition(cond, props)
   if cond == 'all' then return true end

   local p, v = cond:match "(%w+):(%w+)"

   assert(p and v, "failed to parse condition: " .. cond)
   assert(props[p] ~= nil, "undefined configuration property: " .. p)

   return props[p] == v
end

local function BuildFileList(sources, props)
   local list = {}
   for condition, files in pairs(sources) do
      if EvaluateCondition(condition, props) then
235
         for i = 1, #files do table.insert(list, files[i]) end
236 237 238 239 240
      end
   end
   return list
end

241

242 243
local gn_sources = ParseGNFile(false)
local gn_test_sources = ParseGNFile(true)
244

245
local function FilesForArch(arch)
246 247 248 249
   return BuildFileList(gn_sources, { os = 'linux',
                                      arch = arch,
                                      mode = 'debug',
                                      simulator = ''})
250 251
end

252 253 254 255 256 257 258
local function FilesForTest(arch)
   return BuildFileList(gn_test_sources, { os = 'linux',
                                      arch = arch,
                                      mode = 'debug',
                                      simulator = ''})
end

259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
local mtConfig = {}

mtConfig.__index = mtConfig

local function config (t) return setmetatable(t, mtConfig) end

function mtConfig:extend(t)
   local e = {}
   for k, v in pairs(self) do e[k] = v end
   for k, v in pairs(t) do e[k] = v end
   return config(e)
end

local ARCHITECTURES = {
   ia32 = config { triple = "i586-unknown-linux",
274 275
                   arch_define = "V8_TARGET_ARCH_IA32",
                   arch_options = "-m32" },
276
   arm = config { triple = "i586-unknown-linux",
277 278
                  arch_define = "V8_TARGET_ARCH_ARM",
                  arch_options = "-m32" },
279
   x64 = config { triple = "x86_64-unknown-linux",
280 281
                  arch_define = "V8_TARGET_ARCH_X64",
                  arch_options = "" },
282
   arm64 = config { triple = "x86_64-unknown-linux",
283 284
                    arch_define = "V8_TARGET_ARCH_ARM64",
                    arch_options = "" },
285 286 287
}

-------------------------------------------------------------------------------
288 289 290 291 292 293 294 295 296 297
-- GCSuspects Generation

local gc, gc_caused, funcs

local WHITELIST = {
   -- The following functions call CEntryStub which is always present.
   "MacroAssembler.*CallRuntime",
   "CompileCallLoadPropertyWithInterceptor",
   "CallIC.*GenerateMiss",

298
   -- DirectCEntryStub is a special stub used on ARM.
299
   -- It is pinned and always present.
300
   "DirectCEntryStub.*GenerateCall",
301

302 303 304
   -- TODO GCMole currently is sensitive enough to understand that certain
   --      functions only cause GC and return Failure simulataneously.
   --      Callsites of such functions are safe as long as they are properly
305 306
   --      check return value and propagate the Failure to the caller.
   --      It should be possible to extend GCMole to understand this.
307 308 309 310 311 312
   "Heap.*AllocateFunctionPrototype",

   -- Ignore all StateTag methods.
   "StateTag",

   -- Ignore printing of elements transition.
313
   "PrintElementsTransition"
314 315 316 317 318 319 320 321 322 323
};

local function AddCause(name, cause)
   local t = gc_caused[name]
   if not t then
      t = {}
      gc_caused[name] = t
   end
   table.insert(t, cause)
end
324 325 326

local function resolve(name)
   local f = funcs[name]
327 328

   if not f then
329 330
      f = {}
      funcs[name] = f
331 332 333 334 335 336 337 338 339 340 341 342 343

      if name:match "Collect.*Garbage" then
         gc[name] = true
         AddCause(name, "<GC>")
      end

      if FLAGS.whitelist then
         for i = 1, #WHITELIST do
            if name:match(WHITELIST[i]) then
               gc[name] = false
            end
         end
      end
344
   end
345

346 347 348 349 350 351 352 353
    return f
end

local function parse (filename, lines)
   local scope

   for funcname in lines do
      if funcname:sub(1, 1) ~= '\t' then
354 355
         resolve(funcname)
         scope = funcname
356
      else
357 358
         local name = funcname:sub(2)
         resolve(name)[scope] = true
359 360 361 362 363 364 365
      end
   end
end

local function propagate ()
   log "** Propagating GC information"

366 367 368 369 370 371 372
   local function mark(from, callers)
      for caller, _ in pairs(callers) do
         if gc[caller] == nil then
            gc[caller] = true
            mark(caller, funcs[caller])
         end
         AddCause(caller, from)
373 374 375 376
      end
   end

   for funcname, callers in pairs(funcs) do
377
      if gc[funcname] then mark(funcname, callers) end
378 379 380 381
   end
end

local function GenerateGCSuspects(arch, files, cfg)
382 383 384
   -- Reset the global state.
   gc, gc_caused, funcs = {}, {}, {}

385 386 387 388
   log ("** Building GC Suspects for %s", arch)
   InvokeClangPluginForEachFile (files,
                                 cfg:extend { plugin = "dump-callees" },
                                 parse)
389

390 391 392
   propagate()

   local out = assert(io.open("gcsuspects", "w"))
393 394 395 396 397 398 399 400 401 402 403
   for name, value in pairs(gc) do if value then out:write (name, '\n') end end
   out:close()

   local out = assert(io.open("gccauses", "w"))
   out:write "GC = {"
   for name, causes in pairs(gc_caused) do
      out:write("['", name, "'] = {")
      for i = 1, #causes do out:write ("'", causes[i], "';") end
      out:write("};\n")
   end
   out:write "}"
404
   out:close()
405

406 407 408
   log ("** GCSuspects generated for %s", arch)
end

409
--------------------------------------------------------------------------------
410 411
-- Analysis

412 413 414 415 416 417 418
local function CheckCorrectnessForArch(arch, for_test)
   local files
   if for_test then
      files = FilesForTest(arch)
   else
      files = FilesForArch(arch)
   end
419 420
   local cfg = ARCHITECTURES[arch]

421 422 423
   if not FLAGS.reuse_gcsuspects then
      GenerateGCSuspects(arch, files, cfg)
   end
424 425 426

   local processed_files = 0
   local errors_found = false
427
   local output = ""
428 429 430
   local function SearchForErrors(filename, lines)
      processed_files = processed_files + 1
      for l in lines do
431 432 433 434
         errors_found = errors_found or
            l:match "^[^:]+:%d+:%d+:" or
            l:match "error" or
            l:match "warning"
435 436 437 438 439
         if for_test then
            output = output.."\n"..l
         else
            print(l)
         end
440 441 442
      end
   end

443 444 445 446 447
   log("** Searching for evaluation order problems%s for %s",
       FLAGS.dead_vars and " and dead variables" or "",
       arch)
   local plugin_args
   if FLAGS.dead_vars then plugin_args = { "--dead-vars" } end
448
   InvokeClangPluginForEachFile(files,
449 450 451
                                cfg:extend { plugin = "find-problems",
                                             plugin_args = plugin_args },
                                SearchForErrors)
452 453 454 455
   log("** Done processing %d files. %s",
       processed_files,
       errors_found and "Errors found" or "No errors found")

456
   return errors_found, output
457 458
end

459 460
local function SafeCheckCorrectnessForArch(arch, for_test)
   local status, errors, output = pcall(CheckCorrectnessForArch, arch, for_test)
461 462 463 464
   if not status then
      print(string.format("There was an error: %s", errors))
      errors = true
   end
465 466 467 468 469
   return errors, output
end

local function TestRun()
   local errors, output = SafeCheckCorrectnessForArch('x64', true)
470 471 472 473
   if not errors then
      log("** Test file should produce errors, but none were found.")
      return false
   end
474 475 476 477 478 479 480

   local filename = "tools/gcmole/test-expectations.txt"
   local exp_file = assert(io.open(filename), "failed to open test expectations file")
   local expectations = exp_file:read('*all')

   if output ~= expectations then
      log("** Output mismatch from running tests. Please run them manually.")
481
      return false
482
   end
483

484 485 486
   log("** Tests ran successfully")
   return true
end
487

488
local errors = not TestRun()
489 490 491

for _, arch in ipairs(ARCHS) do
   if not ARCHITECTURES[arch] then
492
      error("Unknown arch: " .. arch)
493 494
   end

495
   errors = SafeCheckCorrectnessForArch(arch, false) or errors
496 497 498
end

os.exit(errors and 1 or 0)