If both -f and -D are specified when updating, remove all untracked directories

This is required to avoid the need to clobber the bots when moving a directory
to deps/. Currently, the directory in question is likely to remain in the
working copy, despite having been removed, due to the presence of untracked
files. This causes the checkout from deps/ to fail.

With this change, when both --force and --delete_unversioned_trees are
specified, the the directory in question will be removed from the working copy,
thereby allowing the copy in deps/ to be checked out correctly.

Note that untracked directories which are explicitly ignored (ie in .gitignore
or svn:ignore) will not be removed.

BUG=112887,chromium-os:20759

Review URL: http://codereview.chromium.org/9348054

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@121986 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -1300,10 +1300,14 @@ def CMDsync(parser, args):
                     help='skips any safesync_urls specified in '
                          'configured solutions and sync to head instead')
   parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
-                    help='delete any dependency that have been removed from '
-                         'last sync as long as there is no local modification. '
-                         'Coupled with --force, it will remove them even with '
-                         'local modifications')
+                    help='Deletes from the working copy any dependencies that '
+                         'have been removed since the last sync, as long as '
+                         'there are no local modifications. When used with '
+                         '--force, such dependencies are removed even if they '
+                         'have local modifications. In addition, when used '
+                         'with --force, all untracked directories are removed '
+                         'from the working copy, exclusing those which are '
+                         'explicitly ignored in the repository.')
   parser.add_option('-R', '--reset', action='store_true',
                     help='resets any local changes before updating (git only)')
   parser.add_option('-M', '--merge', action='store_true',
@@ -1368,6 +1372,7 @@ def CMDrevert(parser, args):
   (options, args) = parser.parse_args(args)
   # --force is implied.
   options.force = True
+  options.delete_unversioned_trees = False
   client = GClient.LoadCurrentConfig(options)
   if not client:
     raise gclient_utils.Error('client not configured; see \'gclient config\'')

gclient_scm.py

-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -442,6 +442,20 @@ class GitWrapper(SCMWrapper):
     if verbose:
       print('Checked out revision %s' % self.revinfo(options, (), None))
 
+    # If --force and --delete_unversioned_trees are specified, remove any
+    # untracked directories.
+    if options.force and options.delete_unversioned_trees:
+      # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the
+      # merge-base by default), so doesn't include untracked files. So we use
+      # 'git ls-files --directory --others --exclude-standard' here directly.
+      paths = scm.GIT.Capture(
+          ['ls-files', '--directory', '--others', '--exclude-standard'],
+          self.checkout_path)
+      for path in (p for p in paths.splitlines() if p.endswith('/')):
+        print('\n_____ removing unversioned directory %s' % path)
+        gclient_utils.RemoveDirectory(os.path.join(self.checkout_path, path))
+
+
   def revert(self, options, args, file_list):
     """Reverts local modifications.
 
@@ -914,7 +928,7 @@ class SVNWrapper(SCMWrapper):
         if not options.force and not options.reset:
           # Look for local modifications but ignore unversioned files.
           for status in scm.SVN.CaptureStatus(None, self.checkout_path):
-            if status[0] != '?':
+            if status[0][0] != '?':
               raise gclient_utils.Error(
                   ('Can\'t switch the checkout to %s; UUID don\'t match and '
                    'there is local changes in %s. Delete the directory and '
@@ -939,6 +953,15 @@ class SVNWrapper(SCMWrapper):
     command = self._AddAdditionalUpdateFlags(command, options, revision)
     self._RunAndGetFileList(command, options, file_list, self._root_dir)
 
+    # If --force and --delete_unversioned_trees are specified, remove any
+    # untracked files and directories.
+    if options.force and options.delete_unversioned_trees:
+      for status in scm.SVN.CaptureStatus(None, self.checkout_path):
+        path = os.path.join(self.checkout_path, status[1])
+        if (status[0][0] == '?' and os.path.isdir(path)):
+          print('\n_____ removing unversioned directory %s' % status[1])
+          gclient_utils.RemoveDirectory(os.path.join(path))
+
   def updatesingle(self, options, args, file_list):
     filename = args.pop()
     if scm.SVN.AssertVersion("1.5")[0]:

tests/gclient_scm_test.py

 #!/usr/bin/env python
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -83,6 +83,7 @@ class SVNWrapperTestCase(BaseTestCase):
       self.nohooks = False
       # TODO(maruel): Test --jobs > 1.
       self.jobs = 1
+      self.delete_unversioned_trees = False
 
   def Options(self, *args, **kwargs):
     return self.OptionsObject(*args, **kwargs)
@@ -367,11 +368,102 @@ class SVNWrapperTestCase(BaseTestCase):
     # Cheat a bit here.
     gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
         ).AndReturn(file_info)
+    gclient_scm.scm.SVN.Capture(['--version'], None
+        ).AndReturn('svn, version 1.5.1 (r32289)')
+
     additional_args = []
     if options.manually_grab_svn_rev:
       additional_args = ['--revision', str(file_info['Revision'])]
+    additional_args.extend(['--force', '--ignore-externals'])
+    files_list = []
+    gclient_scm.scm.SVN.RunAndGetFileList(
+        options.verbose,
+        ['update', self.base_path] + additional_args,
+        cwd=self.root_dir, file_list=files_list)
+
+    self.mox.ReplayAll()
+    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                            relpath=self.relpath)
+    scm.update(options, (), files_list)
+
+  def testUpdateForceNoDeleteUnversionedTrees(self):
+    options = self.Options(verbose=True)
+    options.force = True
+
+    file_info = {
+      'Repository Root': 'blah',
+      'URL': self.url,
+      'UUID': 'ABC',
+      'Revision': 42,
+    }
+    gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False)
+    gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False)
+
+    # Create an untracked file and directory.
+    dotted_path = join(self.base_path, '.')
+    gclient_scm.scm.SVN.CaptureStatus(None, dotted_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+
+    # Checkout or update.
+    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
+    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
+    gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
+        ).AndReturn(file_info)
+    gclient_scm.scm.SVN.Capture(['--version'], None
+        ).AndReturn('svn, version 1.5.1 (r32289)')
+
+    additional_args = []
+    if options.manually_grab_svn_rev:
+      additional_args = ['--revision', str(file_info['Revision'])]
+    additional_args.extend(['--force', '--ignore-externals'])
+    files_list = []
+    gclient_scm.scm.SVN.RunAndGetFileList(
+        options.verbose,
+        ['update', self.base_path] + additional_args,
+        cwd=self.root_dir, file_list=files_list)
+
+    self.mox.ReplayAll()
+    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                            relpath=self.relpath)
+    scm.update(options, (), files_list)
+
+  def testUpdateForceDeleteUnversionedTrees(self):
+    options = self.Options(verbose=True)
+    options.force = True
+    options.delete_unversioned_trees = True
+
+    file_info = {
+      'Repository Root': 'blah',
+      'URL': self.url,
+      'UUID': 'ABC',
+      'Revision': 42,
+    }
+    gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False)
+    gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False)
+
+    # Create an untracked file and directory.
+    dotted_path = join(self.base_path, '.')
+    gclient_scm.scm.SVN.CaptureStatus(None, dotted_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+
+    # Checkout or update.
+    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
+    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
+    gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
+        ).AndReturn(file_info)
     gclient_scm.scm.SVN.Capture(['--version'], None
         ).AndReturn('svn, version 1.5.1 (r32289)')
+
+    # Confirm that the untracked file is removed.
+    gclient_scm.scm.SVN.CaptureStatus(None, self.base_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+    gclient_scm.os.path.isdir(join(self.base_path, 'dir')).AndReturn(True)
+    gclient_scm.os.path.isdir(join(self.base_path, 'file')).AndReturn(False)
+    gclient_scm.gclient_utils.RemoveDirectory(join(self.base_path, 'dir'))
+
+    additional_args = []
+    if options.manually_grab_svn_rev:
+      additional_args = ['--revision', str(file_info['Revision'])]
     additional_args.extend(['--force', '--ignore-externals'])
     files_list = []
     gclient_scm.scm.SVN.RunAndGetFileList(
@@ -383,6 +475,7 @@ class SVNWrapperTestCase(BaseTestCase):
     scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
                             relpath=self.relpath)
     scm.update(options, (), files_list)
+    self.checkstdout('\n_____ removing unversioned directory dir\n')
 
   def testUpdateSingleCheckout(self):
     options = self.Options(verbose=True)
@@ -589,6 +682,7 @@ class BaseGitWrapperTestCase(GCBaseTestCase, StdoutCheck, TestCaseUtils,
       self.reset = False
       self.nohooks = False
       self.merge = False
+      self.delete_unversioned_trees = False
 
   sample_git_import = """blob
 mark :1
@@ -895,6 +989,62 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
         'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
         ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n')
 
+  def testUpdateForceNoDeleteUnversionedTrees(self):
+    if not self.enabled:
+      return
+    options = self.Options()
+    options.force = True
+
+    dir_path = join(self.base_path, 'c')
+    os.mkdir(dir_path)
+    open(join(dir_path, 'nested'), 'w').writelines('new\n')
+
+    file_path = join(self.base_path, 'file')
+    open(file_path, 'w').writelines('new\n')
+
+    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
+                                relpath=self.relpath)
+    file_list = []
+    scm.update(options, (), file_list)
+    self.assert_(gclient_scm.os.path.isdir(dir_path))
+    self.assert_(gclient_scm.os.path.isfile(file_path))
+    self.checkstdout(
+        '\n________ running \'git reset --hard HEAD\' in \'%s\''
+        '\nHEAD is now at 069c602 A and B\n'
+        '\n_____ . at refs/heads/master\n'
+        'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
+        ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n'
+        % join(self.root_dir, '.'))
+
+  def testUpdateForceDeleteUnversionedTrees(self):
+    if not self.enabled:
+      return
+    options = self.Options()
+    options.force = True
+    options.delete_unversioned_trees = True
+
+    dir_path = join(self.base_path, 'dir')
+    os.mkdir(dir_path)
+    open(join(dir_path, 'nested'), 'w').writelines('new\n')
+
+    file_path = join(self.base_path, 'file')
+    open(file_path, 'w').writelines('new\n')
+
+    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
+                                relpath=self.relpath)
+    file_list = []
+    scm.update(options, (), file_list)
+    self.assert_(not gclient_scm.os.path.isdir(dir_path))
+    self.assert_(gclient_scm.os.path.isfile(file_path))
+    self.checkstdout(
+        '\n________ running \'git reset --hard HEAD\' in \'%s\''
+        '\nHEAD is now at 069c602 A and B\n'
+        '\n_____ . at refs/heads/master\n'
+        'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
+        ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n'
+        '\n_____ removing unversioned directory dir/\n' % join(self.root_dir,
+                                                               '.'))
+
   def testUpdateUnstagedConflict(self):
     if not self.enabled:
       return