fix(encoding): in untracked_files() and index

* untracked_files could, if there were spaces in the path returned,
  re-rencode the previously decoded unicode string thanks to a
  `decode("string_escape")` call. Now re-encode into utf-8 afterwards
  - added test to assure this works indeed
* IndexFile.add() didn't handle unicode correctly and would write
  broken index files. The solution was to compute the path length after
  encoding it into utf-8 bytes, not before ... .

Closes #320

Author: Byron

git/index/base.py

@@ -583,7 +583,7 @@ def _store_path(self, filepath, fprogress):
         stream = None
         if S_ISLNK(st.st_mode):
             # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8
-            stream = BytesIO(force_bytes(os.readlink(filepath), encoding='utf-8'))
+            stream = BytesIO(force_bytes(os.readlink(filepath), encoding=defenc))
         else:
             stream = open(filepath, 'rb')
         # END handle stream
@@ -610,7 +610,7 @@ def _entries_for_paths(self, paths, path_rewriter, fprogress, entries):
 
                 blob = Blob(self.repo, Blob.NULL_BIN_SHA,
                             stat_mode_to_index_mode(os.stat(abspath).st_mode),
-                            to_native_path_linux(gitrelative_path))
+                            to_native_path_linux(gitrelative_path), encoding=defenc)
                 # TODO: variable undefined
                 entries.append(BaseIndexEntry.from_blob(blob))
             # END for each path

git/index/fun.py

@@ -124,12 +124,13 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1
         write(entry[4])         # ctime
         write(entry[5])         # mtime
         path = entry[3]
+        path = path.encode(defenc)
         plen = len(path) & CE_NAMEMASK      # path length
         assert plen == len(path), "Path %s too long to fit into index" % entry[3]
         flags = plen | (entry[2] & CE_NAMEMASK_INV)     # clear possible previous values
         write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0],
                    entry[8], entry[9], entry[10], entry[1], flags))
-        write(path.encode(defenc))
+        write(path)
         real_size = ((tell() - beginoffset + 8) & ~7)
         write(b"\0" * ((beginoffset + real_size) - tell()))
     # END for each entry

git/repo/base.py

@@ -625,7 +625,7 @@ def _get_untracked_files(self, **kwargs):
             filename = line[len(prefix):].rstrip('\n')
             # Special characters are escaped
             if filename[0] == filename[-1] == '"':
-                filename = filename[1:-1].decode('string_escape')
+                filename = filename[1:-1].decode('string_escape').decode(defenc)
             untracked_files.append(filename)
         finalize_process(proc)
         return untracked_files

git/test/test_repo.py

@@ -1,3 +1,4 @@
+#-*-coding:utf-8-*-
 # test_repo.py
 # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
 #
@@ -324,31 +325,32 @@ def test_blame_complex_revision(self, git):
         assert len(res) == 1
         assert len(res[0][1]) == 83, "Unexpected amount of parsed blame lines"
 
-    def test_untracked_files(self):
-        base = self.rorepo.working_tree_dir
-        files = (join_path_native(base, "__test_myfile"),
-                 join_path_native(base, "__test_other_file"))
-        num_recently_untracked = 0
-        try:
+    @with_rw_repo('HEAD', bare=False)
+    def test_untracked_files(self, rwrepo):
+        for (run, repo_add) in enumerate((rwrepo.index.add, rwrepo.git.add)):
+            base = rwrepo.working_tree_dir
+            files = (join_path_native(base, u"%i_test _myfile" % run),
+                     join_path_native(base, "%i_test_other_file" % run),
+                     join_path_native(base, u"%i__çava verböten" % run),
+                     join_path_native(base, u"%i_çava-----verböten" % run))
+
+            num_recently_untracked = 0
             for fpath in files:
                 fd = open(fpath, "wb")
                 fd.close()
             # END for each filename
-            untracked_files = self.rorepo.untracked_files
+            untracked_files = rwrepo.untracked_files
             num_recently_untracked = len(untracked_files)
 
             # assure we have all names - they are relative to the git-dir
             num_test_untracked = 0
             for utfile in untracked_files:
                 num_test_untracked += join_path_native(base, utfile) in files
             assert len(files) == num_test_untracked
-        finally:
-            for fpath in files:
-                if os.path.isfile(fpath):
-                    os.remove(fpath)
-        # END handle files
 
-        assert len(self.rorepo.untracked_files) == (num_recently_untracked - len(files))
+            repo_add(untracked_files)
+            assert len(rwrepo.untracked_files) == (num_recently_untracked - len(files))
+        # end for each run
 
     def test_config_reader(self):
         reader = self.rorepo.config_reader()                # all config files