Merge branch 'py3' into 0.3

Conflicts:
	git/refs/log.py

Author: Byron

README.md

@@ -83,10 +83,6 @@ In short, I want to make a new release of 0.3 with all contributions and fixes i
 
 The goals I have set for myself, in order, are as follows, all on branch 0.3.
 
-* bring the test suite back online to work with the most commonly used git version
-* merge all open pull requests, may there be a test-case or not, back. If something breaks, fix it if possible or let the contributor know
-* conform git-python's structure and toolchain to the one used in my [other OSS projects](https://github.com/Byron/bcore)
-* evaluate all open issues and close them if possible
 * evaluate python 3.3 compatibility and establish it if possible
 
 While that is happening, I will try hard to foster community around the project. This means being more responsive on the mailing list and in issues, as well as setting up clear guide lines about the [contribution](http://rfc.zeromq.org/spec:22) and maintenance workflow.

VERSION

@@ -1 +1 @@
-0.3.3
+0.3.4

doc/source/changes.rst

@@ -2,10 +2,18 @@
 Changelog
 =========
 
+0.3.4 - Python 3 Support
+========================
+* Internally, hexadecimal SHA1 are treated as ascii encoded strings. Binary SHA1 are treated as bytes.
+* Id attribute of Commit objects is now `hexsha`, instead of `binsha`. The latter makes no sense in python 3 and I see no application of it anyway besides its artificial usage in test cases.
+* **IMPORTANT**: If you were using the config_writer(), you implicitly relied on __del__ to work as expected to flush changes. To be sure changes are flushed under PY3, you will have to call the new `release()` method to trigger a flush. For some reason, __del__ is not called necessarily anymore when a symbol goes out of scope.
+* The `Tree` now has a `.join('name')` method which is equivalent to `tree / 'name'`
+
 0.3.3
 =====
 * When fetching, pulling or pushing, and an error occours, it will not be reported on stdout anymore. However, if there is a fatal error, it will still result in a GitCommandError to be thrown. This goes hand in hand with improved fetch result parsing.
 * Code Cleanup (in preparation for python 3 support)
+
   * Applied autopep8 and cleaned up code
   * Using python logging module instead of print statments to signal certain kinds of errors
 

git/cmd.py

@@ -7,18 +7,24 @@
 import os
 import sys
 import logging
-from util import (
-    LazyMixin,
-    stream_copy
-)
-from exc import GitCommandError
-
 from subprocess import (
     call,
     Popen,
     PIPE
 )
 
+
+from .util import (
+    LazyMixin,
+    stream_copy
+)
+from .exc import GitCommandError
+from git.compat import (
+    string_types,
+    defenc,
+    PY3
+)
+
 execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output',
                   'with_exceptions', 'as_process',
                   'output_stream')
@@ -114,7 +120,7 @@ def wait(self):
             :raise GitCommandError: if the return status is not 0"""
             status = self.proc.wait()
             if status != 0:
-                raise GitCommandError(self.args, status, self.proc.stderr.read())
+                raise GitCommandError(self.args, status, self.proc.stderr.read().decode(defenc))
             # END status handling
             return status
     # END auto interrupt
@@ -314,6 +320,7 @@ def execute(self, command,
             always be created with a pipe due to issues with subprocess.
             This merely is a workaround as data will be copied from the
             output pipe to the given output stream directly.
+            Judging from the implementation, you shouldn't use this flag !
 
         :param subprocess_kwargs:
             Keyword arguments to be passed to subprocess.Popen. Please note that
@@ -365,23 +372,23 @@ def execute(self, command,
 
         # Wait for the process to return
         status = 0
-        stdout_value = ''
-        stderr_value = ''
+        stdout_value = b''
+        stderr_value = b''
         try:
             if output_stream is None:
                 stdout_value, stderr_value = proc.communicate()
                 # strip trailing "\n"
-                if stdout_value.endswith("\n"):
+                if stdout_value.endswith(b"\n"):
                     stdout_value = stdout_value[:-1]
-                if stderr_value.endswith("\n"):
+                if stderr_value.endswith(b"\n"):
                     stderr_value = stderr_value[:-1]
                 status = proc.returncode
             else:
                 stream_copy(proc.stdout, output_stream, self.max_chunk_size)
                 stdout_value = output_stream
                 stderr_value = proc.stderr.read()
                 # strip trailing "\n"
-                if stderr_value.endswith("\n"):
+                if stderr_value.endswith(b"\n"):
                     stderr_value = stderr_value[:-1]
                 status = proc.wait()
             # END stdout handling
@@ -392,9 +399,10 @@ def execute(self, command,
         if self.GIT_PYTHON_TRACE == 'full':
             cmdstr = " ".join(command)
             if stderr_value:
-                log.info("%s -> %d; stdout: '%s'; stderr: '%s'", cmdstr, status, stdout_value, stderr_value)
+                log.info("%s -> %d; stdout: '%s'; stderr: '%s'",
+                         cmdstr, status, stdout_value.decode(defenc), stderr_value.decode(defenc))
             elif stdout_value:
-                log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value)
+                log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value.decode(defenc))
             else:
                 log.info("%s -> %d", cmdstr, status)
         # END handle debug printing
@@ -405,9 +413,12 @@ def execute(self, command,
             else:
                 raise GitCommandError(command, status, stderr_value)
 
+        if isinstance(stdout_value, bytes):  # could also be output_stream
+            stdout_value = stdout_value.decode(defenc)
+
         # Allow access to the command's status code
         if with_extended_output:
-            return (status, stdout_value, stderr_value)
+            return (status, stdout_value, stderr_value.decode(defenc))
         else:
             return stdout_value
 
@@ -433,16 +444,18 @@ def transform_kwargs(self, split_single_char_options=False, **kwargs):
     @classmethod
     def __unpack_args(cls, arg_list):
         if not isinstance(arg_list, (list, tuple)):
-            if isinstance(arg_list, unicode):
-                return [arg_list.encode('utf-8')]
+            # This is just required for unicode conversion, as subprocess can't handle it
+            # However, in any other case, passing strings (usually utf-8 encoded) is totally fine
+            if not PY3 and isinstance(arg_list, unicode):
+                return [arg_list.encode(defenc)]
             return [str(arg_list)]
 
         outlist = list()
         for arg in arg_list:
             if isinstance(arg_list, (list, tuple)):
                 outlist.extend(cls.__unpack_args(arg))
-            elif isinstance(arg_list, unicode):
-                outlist.append(arg_list.encode('utf-8'))
+            elif not PY3 and isinstance(arg_list, unicode):
+                outlist.append(arg_list.encode(defenc))
             # END recursion
             else:
                 outlist.append(str(arg))
@@ -498,8 +511,8 @@ def _call_process(self, method, *args, **kwargs):
 
         # Prepare the argument list
         opt_args = self.transform_kwargs(**kwargs)
-
         ext_args = self.__unpack_args([a for a in args if a is not None])
+
         args = opt_args + ext_args
 
         def make_call():
@@ -567,14 +580,20 @@ def _parse_object_header(self, header_line):
             raise ValueError("Failed to parse header: %r" % header_line)
         return (tokens[0], tokens[1], int(tokens[2]))
 
-    def __prepare_ref(self, ref):
-        # required for command to separate refs on stdin
-        refstr = str(ref)               # could be ref-object
-        if refstr.endswith("\n"):
-            return refstr
-        return refstr + "\n"
+    def _prepare_ref(self, ref):
+        # required for command to separate refs on stdin, as bytes
+        refstr = ref
+        if isinstance(ref, bytes):
+            # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text
+            refstr = ref.decode('ascii')
+        elif not isinstance(ref, string_types):
+            refstr = str(ref)               # could be ref-object
+
+        if not refstr.endswith("\n"):
+            refstr += "\n"
+        return refstr.encode(defenc)
 
-    def __get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs):
+    def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs):
         cur_val = getattr(self, attr_name)
         if cur_val is not None:
             return cur_val
@@ -587,7 +606,7 @@ def __get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs):
         return cmd
 
     def __get_object_header(self, cmd, ref):
-        cmd.stdin.write(self.__prepare_ref(ref))
+        cmd.stdin.write(self._prepare_ref(ref))
         cmd.stdin.flush()
         return self._parse_object_header(cmd.stdout.readline())
 
@@ -599,7 +618,7 @@ def get_object_header(self, ref):
             once and reuses the command in subsequent calls.
 
         :return: (hexsha, type_string, size_as_int)"""
-        cmd = self.__get_persistent_cmd("cat_file_header", "cat_file", batch_check=True)
+        cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True)
         return self.__get_object_header(cmd, ref)
 
     def get_object_data(self, ref):
@@ -616,7 +635,7 @@ def stream_object_data(self, ref):
         :return: (hexsha, type_string, size_as_int, stream)
         :note: This method is not threadsafe, you need one independent  Command instance
             per thread to be safe !"""
-        cmd = self.__get_persistent_cmd("cat_file_all", "cat_file", batch=True)
+        cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True)
         hexsha, typename, size = self.__get_object_header(cmd, ref)
         return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout))
 

git/compat.py

@@ -0,0 +1,66 @@
+#-*-coding:utf-8-*-
+# config.py
+# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors
+#
+# This module is part of GitPython and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php
+"""utilities to help provide compatibility with python 3"""
+# flake8: noqa
+
+import sys
+
+from gitdb.utils.compat import (
+    PY3,
+    xrange,
+    MAXSIZE,
+    izip,
+)
+
+from gitdb.utils.encoding import (
+    string_types,
+    text_type,
+    force_bytes,
+    force_text
+)
+
+defenc = sys.getdefaultencoding()
+if PY3:
+    import io
+    FileType = io.IOBase
+    def byte_ord(b):
+        return b
+    def bchr(n):
+        return bytes([n])
+    def mviter(d):
+        return d.values()
+else:
+    FileType = file
+    # usually, this is just ascii, which might not enough for our encoding needs
+    # Unless it's set specifically, we override it to be utf-8
+    if defenc == 'ascii':
+        defenc = 'utf-8'
+    byte_ord = ord
+    bchr = chr
+    def mviter(d):
+        return d.itervalues()
+
+
+def with_metaclass(meta, *bases):
+    """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15"""
+    class metaclass(meta):
+        __call__ = type.__call__
+        __init__ = type.__init__
+
+        def __new__(cls, name, nbases, d):
+            if nbases is None:
+                return type.__new__(cls, name, (), d)
+            # There may be clients who rely on this attribute to be set to a reasonable value, which is why
+            # we set the __metaclass__ attribute explicitly
+            if not PY3 and '___metaclass__' not in d:
+                d['__metaclass__'] = meta
+            # end
+            return meta(name, bases, d)
+        # end
+    # end metaclass
+    return metaclass(meta.__name__ + 'Helper', None, {})
+    # end handle py2