Fix setup.py and use of requirements files and add a docker test container

.appveyor.yml

@@ -5,8 +5,6 @@ environment:
     CYGWIN64_GIT_PATH: "C:\\cygwin64\\bin;%GIT_DAEMON_PATH%"
 
     matrix:
-        ## MINGW
-        #
       - PYTHON: "C:\\Python27"
         PYTHON_VERSION: "2.7"
         GIT_PATH: "%GIT_DAEMON_PATH%"
@@ -16,24 +14,33 @@ environment:
       - PYTHON: "C:\\Python35-x64"
         PYTHON_VERSION: "3.5"
         GIT_PATH: "%GIT_DAEMON_PATH%"
+      - PYTHON: "C:\\Python36-x64"
+        PYTHON_VERSION: "3.6"
+        GIT_PATH: "%GIT_DAEMON_PATH%"
+      - PYTHON: "C:\\Python37-x64"
+        PYTHON_VERSION: "3.7"
+        GIT_PATH: "%GIT_DAEMON_PATH%"
       - PYTHON: "C:\\Miniconda35-x64"
         PYTHON_VERSION: "3.5"
         IS_CONDA: "yes"
+        MAYFAIL: "yes"
         GIT_PATH: "%GIT_DAEMON_PATH%"
-
         ## Cygwin
-        #
       - PYTHON: "C:\\Miniconda-x64"
         PYTHON_VERSION: "2.7"
         IS_CONDA: "yes"
         IS_CYGWIN: "yes"
+        MAYFAIL: "yes"
         GIT_PATH: "%CYGWIN_GIT_PATH%"
       - PYTHON: "C:\\Python35-x64"
         PYTHON_VERSION: "3.5"
-        GIT_PATH: "%CYGWIN64_GIT_PATH%"
         IS_CYGWIN: "yes"
+        MAYFAIL: "yes"
+        GIT_PATH: "%CYGWIN64_GIT_PATH%"
 
-
+matrix:
+    allow_failures:
+        - MAYFAIL: "yes"
 install:
     - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH%
 
@@ -51,6 +58,7 @@ install:
         conda info -a &
         conda install --yes --quiet  pip
       )
+    - pip install -r requirements.txt
     - pip install -r test-requirements.txt
     - pip install codecov
 

.dockerignore

@@ -0,0 +1,2 @@
+.git/
+.tox/

.travis.yml

@@ -9,12 +9,11 @@ matrix:
   include:
     - python: 3.7
       dist: xenial
-      sudo: required
     - python: "nightly"
       dist: xenial
-      sudo: required
   allow_failures:
     - python: "nightly"
+      dist: xenial
 git:
   # a higher depth is needed for most of the tests - must be high enough to not actually be shallow
   # as we clone our own repository in the process
@@ -40,8 +39,7 @@ script:
   - ulimit -n 128
   - ulimit -n
   - nosetests -v --with-coverage
-  - if [ "$TRAVIS_PYTHON_VERSION" == '3.4' ]; then flake8 --ignore=W293,E265,E266,W503,W504,E731; fi
   - if [ "$TRAVIS_PYTHON_VERSION" == '3.5' ]; then cd doc && make html; fi
-  -
+  - if [ "$TRAVIS_PYTHON_VERSION" == '3.6' ]; then flake8 --ignore=W293,E265,E266,W503,W504,E731; fi
 after_success:
   - codecov

Dockerfile

@@ -0,0 +1,80 @@
+#
+# Contributed by: James E. King III (@jeking3) <jking@apache.org>
+#
+# This Dockerfile creates an Ubuntu Xenial build environment
+# that can run the same test suite as Travis CI.
+#
+
+FROM ubuntu:xenial
+MAINTAINER James E. King III <jking@apache.org>
+ENV CONTAINER_USER=user
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      add-apt-key \
+      apt \
+      apt-transport-https \
+      apt-utils \
+      ca-certificates \
+      curl \
+      git \
+      net-tools \
+      openssh-client \
+      sudo \
+      vim \
+      wget
+
+RUN add-apt-key -v 6A755776 -k keyserver.ubuntu.com && \
+    add-apt-key -v E1DF1F24 -k keyserver.ubuntu.com && \
+    echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \
+    echo "deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y --install-recommends git python2.7 python3.4 python3.5 python3.6 python3.7 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python2.7 27 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.4 34 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 35 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 36 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 37
+
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3 get-pip.py && \
+    pip3 install tox
+
+# Clean up
+RUN rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/* && \
+    rm -rf /var/tmp/*
+
+#################################################################
+# Build as a regular user
+# Credit: https://github.com/delcypher/docker-ubuntu-cxx-dev/blob/master/Dockerfile
+# License: None specified at time of import
+# Add non-root user for container but give it sudo access.
+# Password is the same as the username
+RUN useradd -m ${CONTAINER_USER} && \
+    echo ${CONTAINER_USER}:${CONTAINER_USER} | chpasswd && \
+    echo "${CONTAINER_USER}  ALL=(root) ALL" >> /etc/sudoers
+RUN chsh --shell /bin/bash ${CONTAINER_USER}
+USER ${CONTAINER_USER}
+#################################################################
+
+# The test suite will not tolerate running against a branch that isn't "master", so
+# check out the project to a well-known location that can be used by the test suite.
+# This has the added benefit of protecting the local repo fed into the container
+# as a volume from getting destroyed by a bug exposed by the test suite. :)
+ENV TRAVIS=ON
+RUN git clone --recursive https://github.com/gitpython-developers/GitPython.git /home/${CONTAINER_USER}/testrepo && \
+    cd /home/${CONTAINER_USER}/testrepo && \
+    ./init-tests-after-clone.sh
+ENV GIT_PYTHON_TEST_GIT_REPO_BASE=/home/${CONTAINER_USER}/testrepo
+ENV TRAVIS=
+
+# Ensure any local pip installations get on the path
+ENV PATH=/home/${CONTAINER_USER}/.local/bin:${PATH}
+
+# Set the global default git user to be someone non-descript
+RUN git config --global user.email ci@gitpython.org && \
+    git config --global user.name "GitPython CI User"
+

MANIFEST.in

@@ -2,8 +2,10 @@ include VERSION
 include LICENSE
 include CHANGES
 include AUTHORS
-include README
+include CONTRIBUTING.md
+include README.md
 include requirements.txt
+include test-requirements.txt
 
 recursive-include doc *
 

Makefile

@@ -1,8 +1,10 @@
+.PHONY: all clean release force_release docker-build test nose-pdb
+
 all:
 	@grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all
 
 clean:
-	rm -rf build/ dist/
+	rm -rf build/ dist/ .eggs/ .tox/
 
 release: clean
 	# Check if latest tag is the current head we're releasing
@@ -16,3 +18,17 @@ force_release: clean
 	git push --tags origin master
 	python3 setup.py sdist bdist_wheel
 	twine upload -s -i byronimo@gmail.com dist/*
+
+docker-build:
+	docker build --quiet -t gitpython:xenial -f Dockerfile .
+
+test: docker-build
+	# NOTE!!!
+	# NOTE!!! If you are not running from master or have local changes then tests will fail
+	# NOTE!!!
+	docker run --rm -v ${CURDIR}:/src -w /src -t gitpython:xenial tox
+
+nose-pdb: docker-build
+	# run tests under nose and break on error or failure into python debugger
+	# HINT: set PYVER to "pyXX" to change from the default of py37 to pyXX for nose tests
+	docker run --rm --env PYVER=${PYVER} -v ${CURDIR}:/src -w /src -it gitpython:xenial /bin/bash dockernose.sh

dockernose.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -ex
+if [ -z "${PYVER}" ]; then
+    PYVER=py37
+fi
+
+tox -e ${PYVER} --notest
+PYTHONPATH=/src/.tox/${PYVER}/lib/python*/site-packages /src/.tox/${PYVER}/bin/nosetests --pdb $*

git/cmd.py

@@ -43,6 +43,10 @@
     stream_copy,
 )
 
+try:
+    PermissionError
+except NameError:  # Python < 3.3
+    PermissionError = OSError
 
 execute_kwargs = {'istream', 'with_extended_output',
                   'with_exceptions', 'as_process', 'stdout_as_string',
@@ -211,22 +215,15 @@ def refresh(cls, path=None):
 
         # test if the new git executable path is valid
 
-        if sys.version_info < (3,):
-            # - a GitCommandNotFound error is spawned by ourselves
-            # - a OSError is spawned if the git executable provided
-            #   cannot be executed for whatever reason
-            exceptions = (GitCommandNotFound, OSError)
-        else:
-            # - a GitCommandNotFound error is spawned by ourselves
-            # - a PermissionError is spawned if the git executable provided
-            #   cannot be executed for whatever reason
-            exceptions = (GitCommandNotFound, PermissionError)
-
+        # - a GitCommandNotFound error is spawned by ourselves
+        # - a PermissionError is spawned if the git executable provided
+        #   cannot be executed for whatever reason
+
         has_git = False
         try:
             cls().version()
             has_git = True
-        except exceptions:
+        except (GitCommandNotFound, PermissionError):
             pass
 
         # warn or raise exception if test failed
@@ -718,8 +715,11 @@ def execute(self, command,
         stdout_sink = (PIPE
                        if with_stdout
                        else getattr(subprocess, 'DEVNULL', None) or open(os.devnull, 'wb'))
-        log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s)",
-                  command, cwd, universal_newlines, shell)
+        istream_ok = "None"
+        if istream:
+            istream_ok = "<valid stream>"
+        log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)",
+                  command, cwd, universal_newlines, shell, istream_ok)
         try:
             proc = Popen(command,
                          env=env,

git/remote.py

@@ -695,6 +695,8 @@ def _get_fetch_info_from_stderr(self, proc, progress):
             msg += "Will ignore extra progress lines or fetch head lines."
             msg %= (l_fil, l_fhi)
             log.debug(msg)
+            log.debug("info lines: " + str(fetch_info_lines))
+            log.debug("head info : " + str(fetch_head_info))
             if l_fil < l_fhi:
                 fetch_head_info = fetch_head_info[:l_fil]
             else:

requirements.txt

@@ -1,2 +1 @@
-gitdb>=0.6.4
-ddt>=1.1.1
+gitdb2 (>=2.0.0)

setup.py

@@ -19,6 +19,9 @@
 with open('requirements.txt') as reqs_file:
     requirements = reqs_file.read().splitlines()
 
+with open('test-requirements.txt') as reqs_file:
+    test_requirements = reqs_file.read().splitlines()
+
 
 class build_py(_build_py):
 
@@ -63,10 +66,6 @@ def _stamp_version(filename):
         print("WARNING: Couldn't find version line in file %s" % filename, file=sys.stderr)
 
 
-install_requires = ['gitdb2 >= 2.0.0']
-test_requires = ['ddt>=1.1.1']
-# end
-
 setup(
     name="GitPython",
     cmdclass={'build_py': build_py, 'sdist': sdist},
@@ -81,9 +80,8 @@ def _stamp_version(filename):
     package_dir={'git': 'git'},
     license="BSD License",
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
-    requires=['gitdb2 (>=2.0.0)'],
-    install_requires=install_requires,
-    test_requirements=test_requires + install_requires,
+    requires=requirements,
+    tests_require=requirements + test_requirements,
     zip_safe=False,
     long_description="""GitPython is a python library used to interact with Git repositories""",
     classifiers=[
@@ -110,6 +108,6 @@ def _stamp_version(filename):
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.7"
     ]
 )

test-requirements.txt

@@ -1,6 +1,6 @@
--r requirements.txt
-
+ddt>=1.1.1
 coverage
 flake8
 nose
-mock; python_version=='2.7'
+tox
+mock; python_version=='2.7'