diff --git a/news/6194.bugfix b/news/6194.bugfix
new file mode 100644
index 00000000000..146b0faa5d4
--- /dev/null
+++ b/news/6194.bugfix
@@ -0,0 +1 @@
+Make failed uninstalls roll back more reliably and better at avoiding naming conflicts.
\ No newline at end of file
diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py
index c3596d76478..7f56ae4d7e9 100644
--- a/src/pip/_internal/req/req_uninstall.py
+++ b/src/pip/_internal/req/req_uninstall.py
@@ -15,7 +15,7 @@
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import (
     FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local,
-    normalize_path, renames,
+    normalize_path, renames, rmtree,
 )
 from pip._internal.utils.temp_dir import AdjacentTempDirectory
 
@@ -265,6 +265,13 @@ def remove(self, auto_confirm=False, verbose=False):
                     new_path = self._stash(path)
                     logger.debug('Removing file or directory %s', path)
                     self._moved_paths.append((path, new_path))
+                    if os.path.isdir(path) and os.path.isdir(new_path):
+                        # If we're moving a directory, we need to
+                        # remove the destination first or else it will be
+                        # moved to inside the existing directory.
+                        # We just created new_path ourselves, so it will
+                        # be removable.
+                        os.rmdir(new_path)
                     renames(path, new_path)
                 for pth in self.pth.values():
                     pth.remove()
@@ -311,9 +318,13 @@ def rollback(self):
         logger.info('Rolling back uninstall of %s', self.dist.project_name)
         for path, tmp_path in self._moved_paths:
             logger.debug('Replacing %s', path)
+            if os.path.isdir(tmp_path) and os.path.isdir(path):
+                rmtree(path)
             renames(tmp_path, path)
         for pth in self.pth.values():
             pth.rollback()
+        for save_dir in self._save_dirs:
+            save_dir.cleanup()
 
     def commit(self):
         """Remove temporary save dir: rollback will no longer be possible."""
diff --git a/src/pip/_internal/utils/temp_dir.py b/src/pip/_internal/utils/temp_dir.py
index e7da91675da..9cfed191b26 100644
--- a/src/pip/_internal/utils/temp_dir.py
+++ b/src/pip/_internal/utils/temp_dir.py
@@ -98,7 +98,7 @@ class AdjacentTempDirectory(TempDirectory):
 
     """
     # The characters that may be used to name the temp directory
-    LEADING_CHARS = "-~.+=%0123456789"
+    LEADING_CHARS = "~.+=%0123456789"
 
     def __init__(self, original, delete=None):
         super(AdjacentTempDirectory, self).__init__(delete=delete)