Skip to content

C
Chromium_Depot_tools
Commit 84e43fa2 authored by Takuto Ikuta's avatar Takuto Ikuta Committed by LUCI CQ
Browse files
Options

ninjalog uploader: migrate to python3 

This removes dependency to httplib2 by using urllib.
https://docs.python.org/3/library/http.client.html#httpresponse-objects

Use six for both py2/py3 unittest.

Bug: b/177465438
Change-Id: I48383559842615f97abea45b55ca0acf4d7c8bd3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/2629087
Commit-Queue: Bruce Dawson <brucedawson@chromium.org>
Reviewed-by: 's avatarBruce Dawson <brucedawson@chromium.org>
Reviewed-by: 's avatarYoshisato Yanagisawa <yyanagisawa@google.com>
Reviewed-by: 's avatarFumitoshi Ukai <ukai@google.com>
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
parent 13466d01
Hide whitespace changes
Inline Side-by-side
Showing with 128 additions and 131 deletions
autoninja
View file @ 84e43fa2
......@@ -24,12 +24,12 @@ if eval "$command"; then
fi
# Collect ninjalog from googler.
python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
exit
fi
# Collect ninjalog from googler.
python "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
python3 "$(dirname -- "$0")/ninjalog_uploader_wrapper.py" --cmd $command
# Return an error code of 1 so that if a developer types:
# "autoninja chrome && chrome" then chrome won't run if the build fails.
......
autoninja.bat
View file @ 84e43fa2
......@@ -36,14 +36,14 @@ REM Also print it to reassure that the right settings are being used.
FOR /f "usebackq tokens=*" %%a in (`vpython %scriptdir%autoninja.py "%*"`) do echo %%a & %%a
@if errorlevel 1 goto buildfailure
REM Use call to invoke vpython script here, because we use vpython via vpython.bat.
REM Use call to invoke python script here, because we may use python3 via python3.bat.
@if "%NINJA_SUMMARIZE_BUILD%" == "1" call python3 %scriptdir%post_build_ninja_summary.py %*
@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
exit /b
:buildfailure
@call python.bat %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
@call python3 %scriptdir%ninjalog_uploader_wrapper.py --cmdline %*
REM Return an error code of 1 so that if a developer types:
REM "autoninja chrome && chrome" then chrome won't run if the build fails.
......
ninjalog_uploader.py
View file @ 84e43fa2
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
......@@ -15,8 +15,8 @@ The log will be used to analyze user side build performance.
"""
import argparse
import cStringIO
import gzip
import io
import json
import logging
import multiprocessing
......@@ -26,7 +26,8 @@ import subprocess
import sys
import time
from third_party import httplib2
from third_party.six.moves.urllib import error
from third_party.six.moves.urllib import request
# These build configs affect build performance a lot.
# TODO(https://crbug.com/900161): Add 'blink_symbol_level' and
......@@ -39,10 +40,9 @@ WHITELISTED_CONFIGS = ('symbol_level', 'use_goma', 'is_debug',
def IsGoogler(server):
"""Check whether this script run inside corp network."""
try:
h = httplib2.Http()
_, content = h.request('https://' + server + '/should-upload', 'GET')
return content == 'Success'
except httplib2.HttpLib2Error:
resp = request.urlopen('https://' + server + '/should-upload')
return resp.read() == b'Success'
except error.URLError:
return False
......@@ -202,31 +202,28 @@ def main():
logging.info("ninjalog is not updated recently %s", ninjalog)
return 0
output = cStringIO.StringIO()
output = io.BytesIO()
with open(ninjalog) as f:
with gzip.GzipFile(fileobj=output, mode='wb') as g:
g.write(f.read())
g.write('# end of ninja log\n')
g.write(f.read().encode())
g.write(b'# end of ninja log\n')
metadata = GetMetadata(args.cmdline, ninjalog)
logging.info('send metadata: %s', json.dumps(metadata))
g.write(json.dumps(metadata))
h = httplib2.Http()
resp_headers, content = h.request('https://' + args.server +
'/upload_ninja_log/',
'POST',
body=output.getvalue(),
headers={'Content-Encoding': 'gzip'})
if resp_headers.status != 200:
logging.warning("unexpected status code for response: %s",
resp_headers.status)
g.write(json.dumps(metadata).encode())
resp = request.urlopen(
request.Request('https://' + args.server + '/upload_ninja_log/',
data=output.getvalue(),
headers={'Content-Encoding': 'gzip'}))
if resp.status != 200:
logging.warning("unexpected status code for response: %s", resp.status)
return 1
logging.info('response header: %s', resp_headers)
logging.info('response content: %s', content)
logging.info('response header: %s', resp.headers)
logging.info('response content: %s', resp.read())
return 0
......
ninjalog_uploader_wrapper.py
View file @ 84e43fa2
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
......@@ -115,7 +115,7 @@ def main():
creationnflags = 0
if platform.system() == 'Windows':
creationnflags = subprocess.CREATE_NEW_PROCESS_GROUP
subprocess2.Popen(['vpython', UPLOADER] + sys.argv[1:],
subprocess2.Popen([sys.executable, UPLOADER] + sys.argv[1:],
stdout=devnull,
stderr=devnull,
creationflags=creationnflags)
......
tests/ninjalog_uploader_test.py
View file @ 84e43fa2
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright (c) 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
......@@ -15,105 +15,105 @@ import ninjalog_uploader
class NinjalogUploaderTest(unittest.TestCase):
def test_parse_gn_args(self):
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {})
# Extract current configs from GN's output json.
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
{
'current': {'value': 'true'},
'default': {'value': 'false'},
'name': 'is_component_build'
},
{
'default': {'value': '"x64"'},
'name': 'host_cpu'
},
])), {
'is_component_build': 'true',
'host_cpu': '"x64"',
})
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
{
'current': {'value': 'true'},
'default': {'value': 'false'},
'name': 'is_component_build'
},
{
'current': {'value': 'false'},
'default': {'value': 'false'},
'name': 'use_goma'
},
])), {'is_component_build': 'true',
'use_goma': 'false'})
def test_get_ninjalog(self):
# No args => default to cwd.
self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']),
'./.ninja_log')
# Specified by -C case.
self.assertEqual(
ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']),
'out/Release/.ninja_log')
self.assertEqual(
ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']),
'out/Release/.ninja_log')
# Invalid -C flag case.
self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']),
'./.ninja_log')
# Multiple target directories => use the last directory.
self.assertEqual(ninjalog_uploader.GetNinjalog(
['ninja', '-C', 'out/Release', '-C', 'out/Debug']),
'out/Debug/.ninja_log')
def test_get_build_target_from_command_line(self):
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja']), [])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-j', '1000', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', 'chrome', '-j', '1000']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-C', 'chrome']), [])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-Cout/Release', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all'])
def test_get_j_flag(self):
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', '1000']), 1000)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', '1000a']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', 'a']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j1000']), 1000)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-ja']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j']), None)
def test_parse_gn_args(self):
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([])), {})
# Extract current configs from GN's output json.
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
{
'current': {'value': 'true'},
'default': {'value': 'false'},
'name': 'is_component_build'
},
{
'default': {'value': '"x64"'},
'name': 'host_cpu'
},
])), {
'is_component_build': 'true',
'host_cpu': '"x64"',
})
self.assertEqual(ninjalog_uploader.ParseGNArgs(json.dumps([
{
'current': {'value': 'true'},
'default': {'value': 'false'},
'name': 'is_component_build'
},
{
'current': {'value': 'false'},
'default': {'value': 'false'},
'name': 'use_goma'
},
])), {'is_component_build': 'true',
'use_goma': 'false'})
def test_get_ninjalog(self):
# No args => default to cwd.
self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja']),
'./.ninja_log')
# Specified by -C case.
self.assertEqual(
ninjalog_uploader.GetNinjalog(['ninja', '-C', 'out/Release']),
'out/Release/.ninja_log')
self.assertEqual(
ninjalog_uploader.GetNinjalog(['ninja', '-Cout/Release']),
'out/Release/.ninja_log')
# Invalid -C flag case.
self.assertEqual(ninjalog_uploader.GetNinjalog(['ninja', '-C']),
'./.ninja_log')
# Multiple target directories => use the last directory.
self.assertEqual(ninjalog_uploader.GetNinjalog(
['ninja', '-C', 'out/Release', '-C', 'out/Debug']),
'out/Debug/.ninja_log')
def test_get_build_target_from_command_line(self):
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja']), [])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-j', '1000', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', 'chrome', '-j', '1000']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-C', 'chrome']), [])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-Cout/Release', 'chrome']), ['chrome'])
self.assertEqual(ninjalog_uploader.GetBuildTargetFromCommandLine(
['ninja', '-C', 'out/Release', 'chrome', 'all']), ['chrome', 'all'])
def test_get_j_flag(self):
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', '1000']), 1000)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', '1000a']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j', 'a']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j1000']), 1000)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-ja']), None)
self.assertEqual(ninjalog_uploader.GetJflag(
['ninja','-j']), None)
if __name__ == '__main__':
unittest.main()
unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment