Reland "Make gclient ready for the Blink (DEPS to main project)"

Reland crrev.com/743083002, which was reverted in crrev.com/796053002
due to some test flakiness, probably related with an old version of Git on
the bots. Relanding now that the infra has been updated to Trusty (plus
adding some de-flake precautions).

Original CL Description:
Make gclient ready for the Blink (DEPS to main project) transition

This CL makes gclient understand correctly whether a git project is
being moved from DEPS to an upper project and vice-versa.
The driving use case for this is the upcoming Blink merge, where
third_party/Webkit will be removed from DEPS (and .gitignore) and will
become part of the main project.

At present state, gclient leaves the .git folder around when a project
is removed from DEPS, and that causes many problems.

Furthermore this CL solves the performance problem of bisecting across
the merge point. The subproject's (Blink) .git/ folder is moved to a
backup location (in the main checkout root) and is restored when moving
backwards, avoiding a re-fetch when bisecting across the merge point.

BUG=431469

Review URL: https://codereview.chromium.org/910913003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@294082 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -1542,23 +1542,18 @@ been automagically updated.  The previous version is available at %s.old.
         # Fix path separator on Windows.
         entry_fixed = entry.replace('/', os.path.sep)
         e_dir = os.path.join(self.root_dir, entry_fixed)
-
-        def _IsParentOfAny(parent, path_list):
-          parent_plus_slash = parent + '/'
-          return any(
-              path[:len(parent_plus_slash)] == parent_plus_slash
-              for path in path_list)
-
         # Use entry and not entry_fixed there.
         if (entry not in entries and
             (not any(path.startswith(entry + '/') for path in entries)) and
             os.path.exists(e_dir)):
+          # The entry has been removed from DEPS.
           scm = gclient_scm.CreateSCM(
               prev_url, self.root_dir, entry_fixed, self.outbuf)
 
           # Check to see if this directory is now part of a higher-up checkout.
           # The directory might be part of a git OR svn checkout.
           scm_root = None
+          scm_class = None
           for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN):
             try:
               scm_root = scm_class.GetCheckoutRoot(scm.checkout_path)
@@ -1571,9 +1566,45 @@ been automagically updated.  The previous version is available at %s.old.
                             'determine whether it is part of a higher-level '
                             'checkout, so not removing.' % entry)
             continue
+
+          # This is to handle the case of third_party/WebKit migrating from
+          # being a DEPS entry to being part of the main project.
+          # If the subproject is a Git project, we need to remove its .git
+          # folder. Otherwise git operations on that folder will have different
+          # effects depending on the current working directory.
+          if scm_class == gclient_scm.scm.GIT and (
+              os.path.abspath(scm_root) == os.path.abspath(e_dir)):
+            e_par_dir = os.path.join(e_dir, os.pardir)
+            if scm_class.IsInsideWorkTree(e_par_dir):
+              par_scm_root = scm_class.GetCheckoutRoot(e_par_dir)
+              # rel_e_dir : relative path of entry w.r.t. its parent repo.
+              rel_e_dir = os.path.relpath(e_dir, par_scm_root)
+              if scm_class.IsDirectoryVersioned(par_scm_root, rel_e_dir):
+                save_dir = scm.GetGitBackupDirPath()
+                # Remove any eventual stale backup dir for the same project.
+                if os.path.exists(save_dir):
+                  gclient_utils.rmtree(save_dir)
+                os.rename(os.path.join(e_dir, '.git'), save_dir)
+                # When switching between the two states (entry/ is a subproject
+                # -> entry/ is part of the outer project), it is very likely
+                # that some files are changed in the checkout, unless we are
+                # jumping *exactly* across the commit which changed just DEPS.
+                # In such case we want to cleanup any eventual stale files
+                # (coming from the old subproject) in order to end up with a
+                # clean checkout.
+                scm_class.CleanupDir(par_scm_root, rel_e_dir)
+                assert not os.path.exists(os.path.join(e_dir, '.git'))
+                print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
+                       'level checkout. The git folder containing all the local'
+                       ' branches has been saved to %s.\n'
+                       'If you don\'t care about its state you can safely '
+                       'remove that folder to free up space.') %
+                      (entry, save_dir))
+                continue
+
           if scm_root in full_entries:
-            logging.info('%s is part of a higher level checkout, not '
-                         'removing.', scm.GetCheckoutRoot())
+            logging.info('%s is part of a higher level checkout, not removing',
+                         scm.GetCheckoutRoot())
             continue
 
           file_list = []

gclient_scm.py

@@ -389,6 +389,20 @@ class GitWrapper(SCMWrapper):
     if mirror:
       url = mirror.mirror_path
 
+    # If we are going to introduce a new project, there is a possibility that
+    # we are syncing back to a state where the project was originally a
+    # sub-project rolled by DEPS (realistic case: crossing the Blink merge point
+    # syncing backwards, when Blink was a DEPS entry and not part of src.git).
+    # In such case, we might have a backup of the former .git folder, which can
+    # be used to avoid re-fetching the entire repo again (useful for bisects).
+    backup_dir = self.GetGitBackupDirPath()
+    target_dir = os.path.join(self.checkout_path, '.git')
+    if os.path.exists(backup_dir) and not os.path.exists(target_dir):
+      gclient_utils.safe_makedirs(self.checkout_path)
+      os.rename(backup_dir, target_dir)
+      # Reset to a clean state
+      self._Run(['reset', '--hard', 'HEAD'], options)
+
     if (not os.path.exists(self.checkout_path) or
         (os.path.isdir(self.checkout_path) and
          not os.path.exists(os.path.join(self.checkout_path, '.git')))):
@@ -799,6 +813,12 @@ class GitWrapper(SCMWrapper):
     base_url = self.url
     return base_url[:base_url.rfind('/')] + url
 
+  def GetGitBackupDirPath(self):
+    """Returns the path where the .git folder for the current project can be
+    staged/restored. Use case: subproject moved from DEPS <-> outer project."""
+    return os.path.join(self._root_dir,
+                        'old_' + self.relpath.replace(os.sep, '_')) + '.git'
+
   def _GetMirror(self, url, options):
     """Get a git_cache.Mirror object for the argument url."""
     if not git_cache.Mirror.GetCachePath():

scm.py

@@ -442,6 +442,16 @@ class GIT(object):
     except (OSError, subprocess2.CalledProcessError):
       return False
 
+  @staticmethod
+  def IsDirectoryVersioned(cwd, relative_dir):
+    """Checks whether the given |relative_dir| is part of cwd's repo."""
+    return bool(GIT.Capture(['ls-tree', 'HEAD', relative_dir], cwd=cwd))
+
+  @staticmethod
+  def CleanupDir(cwd, relative_dir):
+    """Cleans up untracked file inside |relative_dir|."""
+    return bool(GIT.Capture(['clean', '-df', relative_dir], cwd=cwd))
+
   @staticmethod
   def GetGitSvnHeadRev(cwd):
     """Gets the most recently pulled git-svn revision."""

testing_support/fake_repos.py

@@ -828,6 +828,40 @@ class FakeRepoSkiaDEPS(FakeReposBase):
     })
 
 
+class FakeRepoBlinkDEPS(FakeReposBase):
+  """Simulates the Blink DEPS transition in Chrome."""
+
+  NB_GIT_REPOS = 2
+  DEPS_pre = 'deps = {"src/third_party/WebKit": "%(git_base)srepo_2",}'
+  DEPS_post = 'deps = {}'
+
+  def populateGit(self):
+    # Blink repo.
+    self._commit_git('repo_2', {
+        'OWNERS': 'OWNERS-pre',
+        'Source/exists_always': '_ignored_',
+        'Source/exists_before_but_not_after': '_ignored_',
+    })
+
+    # Chrome repo.
+    self._commit_git('repo_1', {
+        'DEPS': self.DEPS_pre % {'git_base': self.git_base},
+        'myfile': 'myfile@1',
+        '.gitignore': '/third_party/WebKit',
+    })
+    self._commit_git('repo_1', {
+        'DEPS': self.DEPS_post % {'git_base': self.git_base},
+        'myfile': 'myfile@2',
+        '.gitignore': '',
+        'third_party/WebKit/OWNERS': 'OWNERS-post',
+        'third_party/WebKit/Source/exists_always': '_ignored_',
+        'third_party/WebKit/Source/exists_after_but_not_before': '_ignored',
+    })
+
+  def populateSvn(self):
+    raise NotImplementedError()
+
+
 class FakeReposTestBase(trial_dir.TestCase):
   """This is vaguely inspired by twisted."""
   # Static FakeRepos instances. Lazy loaded.

tests/gclient_scm_test.py

@@ -1245,6 +1245,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
     self.root_dir = '/tmp' if sys.platform != 'win32' else 't:\\tmp'
     self.relpath = 'fake'
     self.base_path = os.path.join(self.root_dir, self.relpath)
+    self.backup_base_path = os.path.join(self.root_dir,
+                                         'old_%s.git' % self.relpath)
 
   def tearDown(self):
     BaseTestCase.tearDown(self)
@@ -1354,6 +1356,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
 
     gclient_scm.os.path.isdir(
         os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
+    gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
     gclient_scm.os.path.exists(self.base_path).AndReturn(True)
     gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
     gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')
@@ -1384,6 +1387,7 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
 
     gclient_scm.os.path.isdir(
         os.path.join(self.base_path, '.git', 'hooks')).AndReturn(False)
+    gclient_scm.os.path.exists(self.backup_base_path).AndReturn(False)
     gclient_scm.os.path.exists(self.base_path).AndReturn(True)
     gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
     gclient_scm.os.path.exists(os.path.join(self.base_path, '.git')

tests/gclient_smoketest.py

@@ -22,7 +22,7 @@ sys.path.insert(0, ROOT_DIR)
 
 from testing_support.fake_repos import join, write
 from testing_support.fake_repos import FakeReposTestBase, FakeRepoTransitive, \
-                                       FakeRepoSkiaDEPS
+                                       FakeRepoSkiaDEPS, FakeRepoBlinkDEPS
 
 import gclient_utils
 import scm as gclient_scm
@@ -1538,6 +1538,138 @@ class SkiaDEPSTransitionSmokeTest(GClientSmokeBase):
                                              skia_src), src_git_url)
 
 
+class BlinkDEPSTransitionSmokeTest(GClientSmokeBase):
+  """Simulate the behavior of bisect bots as they transition across the Blink
+  DEPS change."""
+
+  FAKE_REPOS_CLASS = FakeRepoBlinkDEPS
+
+  def setUp(self):
+    super(BlinkDEPSTransitionSmokeTest, self).setUp()
+    self.enabled = self.FAKE_REPOS.set_up_git()
+    self.checkout_path = os.path.join(self.root_dir, 'src')
+    self.blink = os.path.join(self.checkout_path, 'third_party', 'WebKit')
+    self.blink_git_url = self.FAKE_REPOS.git_base + 'repo_2'
+    self.pre_merge_sha = self.githash('repo_1', 1)
+    self.post_merge_sha = self.githash('repo_1', 2)
+
+  def CheckStatusPreMergePoint(self):
+    self.assertEqual(gclient_scm.GIT.Capture(['config', 'remote.origin.url'],
+                                             self.blink), self.blink_git_url)
+    self.assertTrue(os.path.exists(join(self.blink, '.git')))
+    self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
+    with open(join(self.blink, 'OWNERS')) as f:
+      owners_content = f.read()
+      self.assertEqual('OWNERS-pre', owners_content, 'OWNERS not updated')
+    self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
+    self.assertTrue(os.path.exists(
+        join(self.blink, 'Source', 'exists_before_but_not_after')))
+    self.assertFalse(os.path.exists(
+        join(self.blink, 'Source', 'exists_after_but_not_before')))
+
+  def CheckStatusPostMergePoint(self):
+    # Check that the contents still exists
+    self.assertTrue(os.path.exists(join(self.blink, 'OWNERS')))
+    with open(join(self.blink, 'OWNERS')) as f:
+      owners_content = f.read()
+      self.assertEqual('OWNERS-post', owners_content, 'OWNERS not updated')
+    self.assertTrue(os.path.exists(join(self.blink, 'Source', 'exists_always')))
+    # Check that file removed between the branch point are actually deleted.
+    self.assertTrue(os.path.exists(
+        join(self.blink, 'Source', 'exists_after_but_not_before')))
+    self.assertFalse(os.path.exists(
+        join(self.blink, 'Source', 'exists_before_but_not_after')))
+    # But not the .git folder
+    self.assertFalse(os.path.exists(join(self.blink, '.git')))
+
+  def testBlinkDEPSChangeUsingGclient(self):
+    """Checks that {src,blink} repos are consistent when syncing going back and
+    forth using gclient sync src@revision."""
+    if not self.enabled:
+      return
+
+    self.gclient(['config', '--spec',
+        'solutions=['
+        '{"name": "src",'
+        ' "url": "' + self.git_base + 'repo_1",'
+        '}]'])
+
+    # Go back and forth two times.
+    for _ in xrange(2):
+      res = self.gclient(['sync', '--jobs', '1',
+                          '--revision', 'src@%s' % self.pre_merge_sha])
+      self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+      self.CheckStatusPreMergePoint()
+
+      res = self.gclient(['sync', '--jobs', '1',
+                          '--revision', 'src@%s' % self.post_merge_sha])
+      self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+      self.CheckStatusPostMergePoint()
+
+
+  def testBlinkDEPSChangeUsingGit(self):
+    """Like testBlinkDEPSChangeUsingGclient, but move the main project using
+    directly git and not gclient sync."""
+    if not self.enabled:
+      return
+
+    self.gclient(['config', '--spec',
+        'solutions=['
+        '{"name": "src",'
+        ' "url": "' + self.git_base + 'repo_1",'
+        ' "managed": False,'
+        '}]'])
+
+    # Perform an initial sync to bootstrap the repo.
+    res = self.gclient(['sync', '--jobs', '1'])
+    self.assertEqual(res[2], 0, 'Initial gclient sync failed.')
+
+    # Go back and forth two times.
+    for _ in xrange(2):
+      subprocess2.check_call(['git', 'checkout', '-q', self.pre_merge_sha],
+                             cwd=self.checkout_path)
+      res = self.gclient(['sync', '--jobs', '1'])
+      self.assertEqual(res[2], 0, 'gclient sync failed.')
+      self.CheckStatusPreMergePoint()
+
+      subprocess2.check_call(['git', 'checkout', '-q', self.post_merge_sha],
+                             cwd=self.checkout_path)
+      res = self.gclient(['sync', '--jobs', '1'])
+      self.assertEqual(res[2], 0, 'DEPS change sync failed.')
+      self.CheckStatusPostMergePoint()
+
+
+  def testBlinkLocalBranchesArePreserved(self):
+    """Checks that the state of local git branches are effectively preserved
+    when going back and forth."""
+    if not self.enabled:
+      return
+
+    self.gclient(['config', '--spec',
+        'solutions=['
+        '{"name": "src",'
+        ' "url": "' + self.git_base + 'repo_1",'
+        '}]'])
+
+    # Initialize to pre-merge point.
+    self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
+    self.CheckStatusPreMergePoint()
+
+    # Create a branch named "foo".
+    subprocess2.check_call(['git', 'checkout', '-qB', 'foo'],
+                           cwd=self.blink)
+
+    # Cross the pre-merge point.
+    self.gclient(['sync', '--revision', 'src@%s' % self.post_merge_sha])
+    self.CheckStatusPostMergePoint()
+
+    # Go backwards and check that we still have the foo branch.
+    self.gclient(['sync', '--revision', 'src@%s' % self.pre_merge_sha])
+    self.CheckStatusPreMergePoint()
+    subprocess2.check_call(
+        ['git', 'show-ref', '-q', '--verify', 'refs/heads/foo'], cwd=self.blink)
+
+
 class GClientSmokeFromCheckout(GClientSmokeBase):
   # WebKit abuses this. It has a .gclient and a DEPS from a checkout.
   def setUp(self):

tests/scm_unittest.py

@@ -77,6 +77,7 @@ class GitWrapperTestCase(BaseSCMTestCase):
         'AssertVersion',
         'Capture',
         'CaptureStatus',
+        'CleanupDir',
         'current_version',
         'FetchUpstreamTuple',
         'GenerateDiff',
@@ -92,6 +93,7 @@ class GitWrapperTestCase(BaseSCMTestCase):
         'GetSha1ForSvnRev',
         'GetSVNBranch',
         'GetUpstreamBranch',
+        'IsDirectoryVersioned',
         'IsGitSvn',
         'IsInsideWorkTree',
         'IsValidRevision',