If both -R and -D are specified when updating, remove all untracked directories

This is required to avoid the need to clobber the bots when moving a directory 
to deps/. Currently, the directory in question is likely to remain in the 
working copy, despite having been removed, due to the presence of untracked 
files. This causes the checkout from deps/ to fail. 

With this change, when both --reset and --delete_unversioned_trees are 
specified, the the directory in question will be removed from the working copy, 
thereby allowing the copy in deps/ to be checked out correctly. 

Note that untracked directories which are explicitly ignored (ie in .gitignore 
or svn:ignore) will not be removed. 

Note that this was previously landed in http://codereview.chromium.org/9348054 
but reverted due to problems with symlinks in the chromeos build. 

BUG=112887, chromium-os:20759

Review URL: http://codereview.chromium.org/9404014

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@122300 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -1300,10 +1300,14 @@ def CMDsync(parser, args):
                     help='skips any safesync_urls specified in '
                          'configured solutions and sync to head instead')
   parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
-                    help='delete any dependency that have been removed from '
-                         'last sync as long as there is no local modification. '
-                         'Coupled with --force, it will remove them even with '
-                         'local modifications')
+                    help='Deletes from the working copy any dependencies that '
+                         'have been removed since the last sync, as long as '
+                         'there are no local modifications. When used with '
+                         '--force, such dependencies are removed even if they '
+                         'have local modifications. When used with --reset, '
+                         'all untracked directories are removed from the '
+                         'working copy, exclusing those which are explicitly '
+                         'ignored in the repository.')
   parser.add_option('-R', '--reset', action='store_true',
                     help='resets any local changes before updating (git only)')
   parser.add_option('-M', '--merge', action='store_true',
@@ -1368,6 +1372,8 @@ def CMDrevert(parser, args):
   (options, args) = parser.parse_args(args)
   # --force is implied.
   options.force = True
+  options.reset = False
+  options.delete_unversioned_trees = False
   client = GClient.LoadCurrentConfig(options)
   if not client:
     raise gclient_utils.Error('client not configured; see \'gclient config\'')

gclient_scm.py

@@ -442,6 +442,22 @@ class GitWrapper(SCMWrapper):
     if verbose:
       print('Checked out revision %s' % self.revinfo(options, (), None))
 
+    # If --reset and --delete_unversioned_trees are specified, remove any
+    # untracked directories.
+    if options.reset and options.delete_unversioned_trees:
+      # GIT.CaptureStatus() uses 'dit diff' to compare to a specific SHA1 (the
+      # merge-base by default), so doesn't include untracked files. So we use
+      # 'git ls-files --directory --others --exclude-standard' here directly.
+      paths = scm.GIT.Capture(
+          ['ls-files', '--directory', '--others', '--exclude-standard'],
+          self.checkout_path)
+      for path in (p for p in paths.splitlines() if p.endswith('/')):
+        full_path = os.path.join(self.checkout_path, path)
+        if not os.path.islink(full_path):
+          print('\n_____ removing unversioned directory %s' % path)
+          gclient_utils.RemoveDirectory(full_path)
+
+
   def revert(self, options, args, file_list):
     """Reverts local modifications.
 
@@ -922,7 +938,7 @@ class SVNWrapper(SCMWrapper):
         if not options.force and not options.reset:
           # Look for local modifications but ignore unversioned files.
           for status in scm.SVN.CaptureStatus(None, self.checkout_path):
-            if status[0] != '?':
+            if status[0][0] != '?':
               raise gclient_utils.Error(
                   ('Can\'t switch the checkout to %s; UUID don\'t match and '
                    'there is local changes in %s. Delete the directory and '
@@ -941,11 +957,21 @@ class SVNWrapper(SCMWrapper):
     if not options.force and str(from_info['Revision']) == revision:
       if options.verbose or not forced_revision:
         print('\n_____ %s%s' % (self.relpath, rev_str))
-      return
+    else:
+      command = ['update', self.checkout_path]
+      command = self._AddAdditionalUpdateFlags(command, options, revision)
+      self._RunAndGetFileList(command, options, file_list, self._root_dir)
 
-    command = ['update', self.checkout_path]
-    command = self._AddAdditionalUpdateFlags(command, options, revision)
-    self._RunAndGetFileList(command, options, file_list, self._root_dir)
+    # If --reset and --delete_unversioned_trees are specified, remove any
+    # untracked files and directories.
+    if options.reset and options.delete_unversioned_trees:
+      for status in scm.SVN.CaptureStatus(None, self.checkout_path):
+        full_path = os.path.join(self.checkout_path, status[1])
+        if (status[0][0] == '?'
+            and os.path.isdir(full_path)
+            and not os.path.islink(full_path)):
+          print('\n_____ removing unversioned directory %s' % status[1])
+          gclient_utils.RemoveDirectory(full_path)
 
   def updatesingle(self, options, args, file_list):
     filename = args.pop()

tests/gclient_scm_test.py

@@ -83,6 +83,7 @@ class SVNWrapperTestCase(BaseTestCase):
       self.nohooks = False
       # TODO(maruel): Test --jobs > 1.
       self.jobs = 1
+      self.delete_unversioned_trees = False
 
   def Options(self, *args, **kwargs):
     return self.OptionsObject(*args, **kwargs)
@@ -356,22 +357,26 @@ class SVNWrapperTestCase(BaseTestCase):
     }
     gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False)
     gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False)
+    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
 
-    # Verify no locked files.
+    # Checkout or update.
     dotted_path = join(self.base_path, '.')
+    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
+
+    # Verify no locked files.
     gclient_scm.scm.SVN.CaptureStatus(None, dotted_path).AndReturn([])
 
-    # Checkout or update.
-    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
-    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
     # Cheat a bit here.
     gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
         ).AndReturn(file_info)
+
+    # _AddAdditionalUpdateFlags()
+    gclient_scm.scm.SVN.Capture(['--version'], None
+        ).AndReturn('svn, version 1.5.1 (r32289)')
+
     additional_args = []
     if options.manually_grab_svn_rev:
       additional_args = ['--revision', str(file_info['Revision'])]
-    gclient_scm.scm.SVN.Capture(['--version'], None
-        ).AndReturn('svn, version 1.5.1 (r32289)')
     additional_args.extend(['--force', '--ignore-externals'])
     files_list = []
     gclient_scm.scm.SVN.RunAndGetFileList(
@@ -384,6 +389,80 @@ class SVNWrapperTestCase(BaseTestCase):
                             relpath=self.relpath)
     scm.update(options, (), files_list)
 
+  def testUpdateReset(self):
+    options = self.Options(verbose=True)
+    options.reset = True
+    file_info = {
+      'Repository Root': 'blah',
+      'URL': self.url,
+      'UUID': 'ABC',
+      'Revision': 42,
+    }
+    gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False)
+    gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False)
+    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
+
+    # Checkout or update.
+    dotted_path = join(self.base_path, '.')
+    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
+
+    # Create an untracked file and directory.
+    gclient_scm.scm.SVN.CaptureStatus(None, dotted_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+
+    gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
+        ).AndReturn(file_info)
+
+    self.mox.ReplayAll()
+    files_list = []
+    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                            relpath=self.relpath)
+    scm.update(options, (), files_list)
+    self.checkstdout('\n_____ %s at 42\n' % self.relpath)
+
+  def testUpdateResetDeleteUnversionedTrees(self):
+    options = self.Options(verbose=True)
+    options.reset = True
+    options.delete_unversioned_trees = True
+
+    file_info = {
+      'Repository Root': 'blah',
+      'URL': self.url,
+      'UUID': 'ABC',
+      'Revision': 42,
+    }
+    gclient_scm.os.path.exists(join(self.base_path, '.git')).AndReturn(False)
+    gclient_scm.os.path.exists(join(self.base_path, '.hg')).AndReturn(False)
+    gclient_scm.os.path.exists(self.base_path).AndReturn(True)
+
+    # Checkout or update.
+    dotted_path = join(self.base_path, '.')
+    gclient_scm.scm.SVN._CaptureInfo([], dotted_path).AndReturn(file_info)
+
+    # Create an untracked file and directory.
+    gclient_scm.scm.SVN.CaptureStatus(None, dotted_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+
+    gclient_scm.scm.SVN._CaptureInfo([file_info['URL']], None
+        ).AndReturn(file_info)
+
+    # Confirm that the untracked file is removed.
+    gclient_scm.scm.SVN.CaptureStatus(None, self.base_path
+        ).AndReturn([['?  ', 'dir'], ['?  ', 'file']])
+    gclient_scm.os.path.isdir(join(self.base_path, 'dir')).AndReturn(True)
+    gclient_scm.os.path.isdir(join(self.base_path, 'file')).AndReturn(False)
+    gclient_scm.os.path.islink(join(self.base_path, 'dir')).AndReturn(False)
+    gclient_scm.gclient_utils.RemoveDirectory(join(self.base_path, 'dir'))
+
+    self.mox.ReplayAll()
+    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                            relpath=self.relpath)
+    files_list = []
+    scm.update(options, (), files_list)
+    self.checkstdout(
+      ('\n_____ %s at 42\n'
+       '\n_____ removing unversioned directory dir\n') % self.relpath)
+
   def testUpdateSingleCheckout(self):
     options = self.Options(verbose=True)
     file_info = {
@@ -589,6 +668,7 @@ class BaseGitWrapperTestCase(GCBaseTestCase, StdoutCheck, TestCaseUtils,
       self.reset = False
       self.nohooks = False
       self.merge = False
+      self.delete_unversioned_trees = False
 
   sample_git_import = """blob
 mark :1
@@ -895,6 +975,62 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
         'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
         ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n')
 
+  def testUpdateReset(self):
+    if not self.enabled:
+      return
+    options = self.Options()
+    options.reset = True
+
+    dir_path = join(self.base_path, 'c')
+    os.mkdir(dir_path)
+    open(join(dir_path, 'nested'), 'w').writelines('new\n')
+
+    file_path = join(self.base_path, 'file')
+    open(file_path, 'w').writelines('new\n')
+
+    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
+                                relpath=self.relpath)
+    file_list = []
+    scm.update(options, (), file_list)
+    self.assert_(gclient_scm.os.path.isdir(dir_path))
+    self.assert_(gclient_scm.os.path.isfile(file_path))
+    self.checkstdout(
+        '\n________ running \'git reset --hard HEAD\' in \'%s\''
+        '\nHEAD is now at 069c602 A and B\n'
+        '\n_____ . at refs/heads/master\n'
+        'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
+        ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n'
+        % join(self.root_dir, '.'))
+
+  def testUpdateResetDeleteUnversionedTrees(self):
+    if not self.enabled:
+      return
+    options = self.Options()
+    options.reset = True
+    options.delete_unversioned_trees = True
+
+    dir_path = join(self.base_path, 'dir')
+    os.mkdir(dir_path)
+    open(join(dir_path, 'nested'), 'w').writelines('new\n')
+
+    file_path = join(self.base_path, 'file')
+    open(file_path, 'w').writelines('new\n')
+
+    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
+                                relpath=self.relpath)
+    file_list = []
+    scm.update(options, (), file_list)
+    self.assert_(not gclient_scm.os.path.isdir(dir_path))
+    self.assert_(gclient_scm.os.path.isfile(file_path))
+    self.checkstdout(
+        '\n________ running \'git reset --hard HEAD\' in \'%s\''
+        '\nHEAD is now at 069c602 A and B\n'
+        '\n_____ . at refs/heads/master\n'
+        'Updating 069c602..a7142dc\nFast-forward\n a |    1 +\n b |    1 +\n'
+        ' 2 files changed, 2 insertions(+), 0 deletions(-)\n\n'
+        '\n_____ removing unversioned directory dir/\n' % join(self.root_dir,
+                                                               '.'))
+
   def testUpdateUnstagedConflict(self):
     if not self.enabled:
       return