fix(cmd): improve character encoding detection for sub-commands

Author: jenstroeger

commitizen/cmd.py

@@ -3,6 +3,8 @@
 
 from charset_normalizer import from_bytes
 
+from commitizen.exceptions import CharacterSetDecodeError
+
 
 class Command(NamedTuple):
     out: str
@@ -12,6 +14,19 @@ class Command(NamedTuple):
     return_code: int
 
 
+def _try_decode(bytes_: bytes) -> str:
+    try:
+        return bytes_.decode("utf-8")
+    except UnicodeDecodeError:
+        charset_match = from_bytes(bytes_).best()
+        if charset_match is None:
+            raise CharacterSetDecodeError()
+        try:
+            return bytes_.decode(charset_match.encoding)
+        except UnicodeDecodeError as e:
+            raise CharacterSetDecodeError() from e
+
+
 def run(cmd: str) -> Command:
     process = subprocess.Popen(
         cmd,
@@ -23,8 +38,8 @@ def run(cmd: str) -> Command:
     stdout, stderr = process.communicate()
     return_code = process.returncode
     return Command(
-        str(from_bytes(stdout).best()),
-        str(from_bytes(stderr).best()),
+        _try_decode(stdout),
+        _try_decode(stderr),
         stdout,
         stderr,
         return_code,

commitizen/exceptions.py

@@ -26,6 +26,7 @@ class ExitCode(enum.IntEnum):
     INVALID_CONFIGURATION = 19
     NOT_ALLOWED = 20
     NO_INCREMENT = 21
+    UNRECOGNIZED_CHARACTERSET_ENCODING = 22
 
 
 class CommitizenException(Exception):
@@ -148,3 +149,7 @@ class InvalidConfigurationError(CommitizenException):
 
 class NotAllowed(CommitizenException):
     exit_code = ExitCode.NOT_ALLOWED
+
+
+class CharacterSetDecodeError(CommitizenException):
+    exit_code = ExitCode.UNRECOGNIZED_CHARACTERSET_ENCODING

docs/exit_codes.md

@@ -28,4 +28,5 @@ These exit codes can be found in `commitizen/exceptions.py::ExitCode`.
 | InvalidCommandArgumentError | 18        | The argument provide to command is invalid (e.g. `cz check -commit-msg-file filename --rev-range master..`) |
 | InvalidConfigurationError   | 19        | An error was found in the Commitizen Configuration, such as duplicates in `change_type_order`               |
 | NotAllowed                  | 20        | `--incremental` cannot be combined with a `rev_range`                                                       |
-| NoneIncrementExit           | 21        | The commits found are not elegible to be bumped                                                             |
+| NoneIncrementExit           | 21        | The commits found are not eligible to be bumped                                                             |
+| CharacterSetDecodeError     | 22        | The character encoding of the command output could not be determined                                        |

tests/test_cmd.py

@@ -0,0 +1,53 @@
+import pytest
+
+from commitizen import cmd
+from commitizen.exceptions import CharacterSetDecodeError
+
+
+# https://docs.python.org/3/howto/unicode.html
+def test_valid_utf8_encoded_strings():
+    valid_strings = (
+        "",
+        "ascii",
+        "🤦🏻‍♂️",
+        "﷽",
+        "\u0000",
+    )
+    assert all(s == cmd._try_decode(s.encode("utf-8")) for s in valid_strings)
+
+
+# A word of caution: just because an encoding can be guessed for a given
+# sequence of bytes and because that guessed encoding may yield a decoded
+# string, does not mean that that string was the original! For more, see:
+# https://docs.python.org/3/library/codecs.html#standard-encodings
+
+
+# Pick a random, non-utf8 encoding to test.
+def test_valid_cp1250_encoded_strings():
+    valid_strings = (
+        "",
+        "ascii",
+        "äöüß",
+        "ça va",
+        "jak se máte",
+    )
+    for s in valid_strings:
+        assert cmd._try_decode(s.encode("cp1250")) or True
+
+
+def test_invalid_bytes():
+    invalid_bytes = (b"\x73\xe2\x9d\xff\x00",)
+    for s in invalid_bytes:
+        with pytest.raises(CharacterSetDecodeError):
+            cmd._try_decode(s)
+
+
+def test_always_fail_decode():
+    class _bytes(bytes):
+        def decode(self, encoding="utf-8", errors="strict"):
+            raise UnicodeDecodeError(
+                encoding, self, 0, 0, "Failing intentionally for testing"
+            )
+
+    with pytest.raises(CharacterSetDecodeError):
+        cmd._try_decode(_bytes())