Add support in gclient for pre-DEPS hooks

These are run for a given dependency after it has been synced but before its
DEPS have been synced. This will help to switch Chromium to depend on Skia's
git repository (skia:1638).

Review URL: https://codereview.chromium.org/25322002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@228651 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -55,6 +55,12 @@
 #         "action":  ["python", "src/build/gyp_chromium"]},
 #     ]
 #
+# Pre-DEPS Hooks
+#   DEPS files may optionally contain a list named "pre_deps_hooks".  These are
+#   the same as normal hooks, except that they run before the DEPS are
+#   processed. Pre-DEPS run with "sync" and "revert" unless the --noprehooks
+#   flag is used.
+# 
 # Specifying a target OS
 #   An optional key named "target_os" may be added to a gclient file to specify
 #   one or more additional operating systems that should be considered when
@@ -286,6 +292,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     # This is in both .gclient and DEPS files:
     self._deps_hooks = []
 
+    self._pre_deps_hooks = []
+
     # Calculates properties:
     self._parsed_url = None
     self._dependencies = []
@@ -297,6 +305,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     self._deps_parsed = False
     # This dependency has been processed, i.e. checked out
     self._processed = False
+    # This dependency had its pre-DEPS hooks run
+    self._pre_deps_hooks_ran = False
     # This dependency had its hook run
     self._hooks_ran = False
     # This is the scm used to checkout self.url. It may be used by dependencies
@@ -548,6 +558,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
       if 'action' in hook:
         hooks_to_run.append(hook)
 
+    self._pre_deps_hooks = [self.GetHookAction(hook, []) for hook in
+                            local_scope.get('pre_deps_hooks', [])]
+
     self.add_dependencies_and_close(deps_to_add, hooks_to_run)
     logging.info('ParseDepsFile(%s) done' % self.name)
 
@@ -648,8 +661,9 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
 
     # Always parse the DEPS file.
     self.ParseDepsFile()
-
     self._run_is_done(file_list or [], parsed_url)
+    if command in ('update', 'revert') and not options.noprehooks:
+      self.RunPreDepsHooks()
 
     if self.recursion_limit:
       # Parse the dependencies of this dependency.
@@ -791,6 +805,32 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
           print "Hook '%s' took %.2f secs" % (
               gclient_utils.CommandToStr(hook), elapsed_time)
 
+  def RunPreDepsHooks(self):
+    assert self.processed
+    assert self.deps_parsed
+    assert not self.pre_deps_hooks_ran
+    assert not self.hooks_ran
+    for s in self.dependencies:
+      assert not s.processed
+    self._pre_deps_hooks_ran = True
+    for hook in self.pre_deps_hooks:
+      try:
+        start_time = time.time()
+        gclient_utils.CheckCallAndFilterAndHeader(
+            hook, cwd=self.root.root_dir, always=True)
+      except (gclient_utils.Error, subprocess2.CalledProcessError), e:
+        # Use a discrete exit status code of 2 to indicate that a hook action
+        # failed.  Users of this script may wish to treat hook action failures
+        # differently from VC failures.
+        print >> sys.stderr, 'Error: %s' % str(e)
+        sys.exit(2)
+      finally:
+        elapsed_time = time.time() - start_time
+        if elapsed_time > 10:
+          print "Hook '%s' took %.2f secs" % (
+              gclient_utils.CommandToStr(hook), elapsed_time)
+
+
   def subtree(self, include_all):
     """Breadth first recursion excluding root node."""
     dependencies = self.dependencies
@@ -828,6 +868,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
   def deps_hooks(self):
     return tuple(self._deps_hooks)
 
+  @property
+  @gclient_utils.lockedmethod
+  def pre_deps_hooks(self):
+    return tuple(self._pre_deps_hooks)
+
   @property
   @gclient_utils.lockedmethod
   def parsed_url(self):
@@ -844,6 +889,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
   def processed(self):
     return self._processed
 
+  @property
+  @gclient_utils.lockedmethod
+  def pre_deps_hooks_ran(self):
+    return self._pre_deps_hooks_ran
+
   @property
   @gclient_utils.lockedmethod
   def hooks_ran(self):
@@ -1555,6 +1605,8 @@ def CMDsync(parser, args):
                     help='force update even for unchanged modules')
   parser.add_option('-n', '--nohooks', action='store_true',
                     help='don\'t run hooks after the update is complete')
+  parser.add_option('-p', '--noprehooks', action='store_true',
+                    help='don\'t run pre-DEPS hooks', default=False)
   parser.add_option('-r', '--revision', action='append',
                     dest='revisions', metavar='REV', default=[],
                     help='Enforces revision/hash for the solutions with the '
@@ -1660,6 +1712,8 @@ def CMDrevert(parser, args):
                          'references')
   parser.add_option('-n', '--nohooks', action='store_true',
                     help='don\'t run hooks after the revert is complete')
+  parser.add_option('-p', '--noprehooks', action='store_true',
+                    help='don\'t run pre-DEPS hooks', default=False)
   parser.add_option('--upstream', action='store_true',
                     help='Make repo state match upstream branch.')
   (options, args) = parser.parse_args(args)
@@ -1793,6 +1847,8 @@ class OptionParser(optparse.OptionParser):
       options.head = None
     if not hasattr(options, 'nohooks'):
       options.nohooks = True
+    if not hasattr(options, 'noprehooks'):
+      options.noprehooks = True
     if not hasattr(options, 'deps_os'):
       options.deps_os = None
     if not hasattr(options, 'manually_grab_svn_rev'):

testing_support/fake_repos.py

@@ -439,7 +439,7 @@ class FakeReposBase(object):
 
 class FakeRepos(FakeReposBase):
   """Implements populateSvn() and populateGit()."""
-  NB_GIT_REPOS = 4
+  NB_GIT_REPOS = 5
 
   def populateSvn(self):
     """Creates a few revisions of changes including DEPS files."""
@@ -559,7 +559,7 @@ hooks = [
 
   def populateGit(self):
     # Testing:
-    # - dependency disapear
+    # - dependency disappear
     # - dependency renamed
     # - versioned and unversioned reference
     # - relative and full reference
@@ -657,6 +657,55 @@ hooks = [
       'origin': 'git/repo_1@2\n',
     })
 
+    self._commit_git('repo_5', {'origin': 'git/repo_5@1\n'})
+    self._commit_git('repo_5', {
+      'DEPS': """
+deps = {
+  'src/repo1': '%(git_base)srepo_1@%(hash1)s',
+  'src/repo2': '%(git_base)srepo_2@%(hash2)s',
+}
+
+# Hooks to run after a project is processed but before its dependencies are
+# processed.
+pre_deps_hooks = [
+  {
+    'action': ['python', '-c',
+               'print "pre-deps hook"; open(\\'src/git_pre_deps_hooked\\', \\'w\\').write(\\'git_pre_deps_hooked\\')'],
+  }
+]
+""" % {
+         'git_base': self.git_base,
+         'hash1': self.git_hashes['repo_1'][2][0][:7],
+         'hash2': self.git_hashes['repo_2'][1][0][:7],
+      },
+    'origin': 'git/repo_5@2\n',
+    })
+    self._commit_git('repo_5', {
+      'DEPS': """
+deps = {
+  'src/repo1': '%(git_base)srepo_1@%(hash1)s',
+  'src/repo2': '%(git_base)srepo_2@%(hash2)s',
+}
+
+# Hooks to run after a project is processed but before its dependencies are
+# processed.
+pre_deps_hooks = [
+  {
+    'action': ['python', '-c',
+               'print "pre-deps hook"; open(\\'src/git_pre_deps_hooked\\', \\'w\\').write(\\'git_pre_deps_hooked\\')'],
+  },
+  {
+    'action': ['python', '-c', 'import sys; sys.exit(1)'],
+  }
+]
+""" % {
+         'git_base': self.git_base,
+         'hash1': self.git_hashes['repo_1'][2][0][:7],
+         'hash2': self.git_hashes['repo_2'][1][0][:7],
+      },
+    'origin': 'git/repo_5@3\n',
+    })
+
 
 class FakeRepoTransitive(FakeReposBase):
   """Implements populateSvn()"""

tests/gclient_smoketest.py

@@ -1067,6 +1067,82 @@ class GClientSmokeGIT(GClientSmokeBase):
     tree['src/git_hooked2'] = 'git_hooked2'
     self.assertTree(tree)
 
+  def testPreDepsHooks(self):
+    if not self.enabled:
+      return
+    self.gclient(['config', self.git_base + 'repo_5', '--name', 'src'])
+    expectation = [
+        ('running', self.root_dir),                 # git clone repo_5
+        ('running', self.root_dir + '/src'),        # git checkout src
+        ('running', self.root_dir),                 # pre-deps hook
+        ('running', self.root_dir),                 # git clone repo_1
+        ('running', self.root_dir + '/src/repo1'),  # git checkout repo1
+        ('running', self.root_dir),                 # git clone repo_1
+        ('running', self.root_dir + '/src/repo2'),  # git checkout repo2
+    ]
+    out = self.parseGclient(['sync', '--deps', 'mac', '--jobs=1',
+                             '--revision', 'src@' + self.githash('repo_5', 2)],
+                            expectation)
+    self.assertEquals(2, len(out[2]))
+    self.assertEquals('pre-deps hook', out[2][1])
+    tree = self.mangle_git_tree(('repo_5@2', 'src'),
+                                ('repo_1@2', 'src/repo1'),
+                                ('repo_2@1', 'src/repo2')
+                                )
+    tree['src/git_pre_deps_hooked'] = 'git_pre_deps_hooked'
+    self.assertTree(tree)
+
+    os.remove(join(self.root_dir, 'src', 'git_pre_deps_hooked'))
+
+    # Pre-DEPS hooks don't run with runhooks.
+    self.gclient(['runhooks', '--deps', 'mac'])
+    tree = self.mangle_git_tree(('repo_5@2', 'src'),
+                                ('repo_1@2', 'src/repo1'),
+                                ('repo_2@1', 'src/repo2')
+                                )
+    self.assertTree(tree)
+
+    # Pre-DEPS hooks run when syncing with --nohooks.
+    self.gclient(['sync', '--deps', 'mac', '--nohooks',
+                  '--revision', 'src@' + self.githash('repo_5', 2)])
+    tree = self.mangle_git_tree(('repo_5@2', 'src'),
+                                ('repo_1@2', 'src/repo1'),
+                                ('repo_2@1', 'src/repo2')
+                                )
+    tree['src/git_pre_deps_hooked'] = 'git_pre_deps_hooked'
+    self.assertTree(tree)
+
+    os.remove(join(self.root_dir, 'src', 'git_pre_deps_hooked'))
+
+    # Pre-DEPS hooks don't run with --noprehooks
+    self.gclient(['sync', '--deps', 'mac', '--noprehooks',
+                  '--revision', 'src@' + self.githash('repo_5', 2)])
+    tree = self.mangle_git_tree(('repo_5@2', 'src'),
+                                ('repo_1@2', 'src/repo1'),
+                                ('repo_2@1', 'src/repo2')
+                                )
+    self.assertTree(tree)
+
+  def testPreDepsHooksError(self):
+    if not self.enabled:
+      return
+    self.gclient(['config', self.git_base + 'repo_5', '--name', 'src'])
+    expectated_stdout = [
+        ('running', self.root_dir),                 # git clone repo_5
+        ('running', self.root_dir + '/src'),        # git checkout src
+        ('running', self.root_dir),                 # pre-deps hook
+        ('running', self.root_dir),                 # pre-deps hook (fails)
+    ]
+    expected_stderr = ('Error: Command /usr/bin/python -c import sys; '
+                       'sys.exit(1) returned non-zero exit status 1 in %s\n'
+                       % self.root_dir)
+    stdout, stderr, retcode = self.gclient(['sync', '--deps', 'mac', '--jobs=1',
+                                            '--revision',
+                                            'src@' + self.githash('repo_5', 3)])
+    self.assertEquals(stderr, expected_stderr)
+    self.assertEquals(2, retcode)
+    self.checkBlock(stdout, expectated_stdout)
+
   def testRevInfo(self):
     if not self.enabled:
       return