General fixes for next release

.github/workflows/pythonpackage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10.0-beta.4"]
+        python-version: [3.7, 3.7.5, 3.7.12, 3.8, 3.8.0, 3.8.11, 3.8, 3.9, 3.9.0, 3.9.7] # , "3.10.0-rc.2"]
 
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -22,3 +22,4 @@ nbproject
 .pytest_cache/
 monkeytype.sqlite3
 output.txt
+tox.ini

VERSION

@@ -1 +1 @@
-3.1.20
+3.1.21

doc/requirements.txt

@@ -1,3 +1,3 @@
-sphinx==4.1.1
+sphinx==4.1.2
 sphinx_rtd_theme
 sphinx-autodoc-typehints

doc/source/changes.rst

@@ -2,32 +2,60 @@
 Changelog
 =========
 
-3.1.20
+3.1.21
 ======
 
-* This is the second typed release with a lot of improvements under the hood. 
+* This is the second typed release with a lot of improvements under the hood.
+
+* General:
+  - Remove python 3.6 support
+  - Remove distutils ahead of deprecation in standard library.
+  - Update sphinx to 4.1.12 and use autodoc-typehints.
+
+* Typing:
+  - Add types to ALL functions.
+  - Ensure py.typed is collected.
+  - Increase mypy strictness with disallow_untyped_defs, warn_redundant_casts, warn_unreachable.
+  - Use typing.NamedTuple and typing.OrderedDict now 3.6 dropped.
+  - Make Protocol classes ABCs at runtime due to new bug in 3.10.0-rc1
+  - Remove use of typing.TypeGuard until later release, to allow dependant libs time to update.
+  - Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095
+
+* Runtime improvements:
+  - Add clone_multi_options support to submodule.add()
+  - Delay calling get_user_id() unless essential, to support sand-boxed environments.
+  - Add timeout to handle_process_output(), in case thread.join() hangs.
+
+See the following for details:
+https://github.com/gitpython-developers/gitpython/milestone/52?closed=1
+
+
+3.1.20 (YANKED)
+===============
+
+* This is the second typed release with a lot of improvements under the hood.
   * Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095
-  
+
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/52?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/52?closed=1
 
 
 3.1.19 (YANKED)
 ===============
 
-* This is the second typed release with a lot of improvements under the hood. 
+* This is the second typed release with a lot of improvements under the hood.
   * Tracking issue: https://github.com/gitpython-developers/GitPython/issues/1095
-  
+
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/51?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/51?closed=1
 
 3.1.18
 ======
 
 * drop support for python 3.5 to reduce maintenance burden on typing. Lower patch levels of python 3.5 would break, too.
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/50?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/50?closed=1
 
 3.1.17
 ======
@@ -37,7 +65,7 @@ https://github.com/gitpython-developers/gitpython/milestone/50?closed=1
 * Add more static typing information
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/49?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/49?closed=1
 
 3.1.16 (YANKED)
 ===============
@@ -46,15 +74,15 @@ https://github.com/gitpython-developers/gitpython/milestone/49?closed=1
 * Add more static typing information
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/48?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/48?closed=1
 
 3.1.15 (YANKED)
 ===============
 
 * add deprectation warning for python 3.5
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/47?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/47?closed=1
 
 3.1.14
 ======
@@ -65,40 +93,40 @@ https://github.com/gitpython-developers/gitpython/milestone/47?closed=1
 * Drop python 3.4 support
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/46?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/46?closed=1
 
 3.1.13
 ======
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/45?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/45?closed=1
 
 3.1.12
 ======
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/44?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/44?closed=1
 
 3.1.11
 ======
 
 Fixes regression of 3.1.10.
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/43?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/43?closed=1
 
 3.1.10
 ======
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/42?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/42?closed=1
 
 
 3.1.9
 =====
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/41?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/41?closed=1
 
 
 3.1.8
@@ -109,7 +137,7 @@ https://github.com/gitpython-developers/gitpython/milestone/41?closed=1
 
 
 See the following for more details:
-https://github.com/gitpython-developers/gitpython/milestone/40?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/40?closed=1
 
 
 3.1.7
@@ -135,13 +163,13 @@ https://github.com/gitpython-developers/gitpython/milestone/40?closed=1
 * package size was reduced significantly not placing tests into the package anymore.
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/39?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/39?closed=1
 
 3.1.3
 =====
 
 See the following for details:
-https://github.com/gitpython-developers/gitpython/milestone/38?closed=1 
+https://github.com/gitpython-developers/gitpython/milestone/38?closed=1
 
 3.1.2
 =====
@@ -190,7 +218,7 @@ Bugfixes
 Bugfixes
 --------
 
-* Fixed Repo.__repr__ when subclassed 
+* Fixed Repo.__repr__ when subclassed
   (`#968 <https://github.com/gitpython-developers/GitPython/pull/968>`_)
 * Removed compatibility shims for Python < 3.4 and old mock library
 * Replaced usage of deprecated unittest aliases and Logger.warn
@@ -213,7 +241,7 @@ Bugfixes
 --------
 
 * Fixed warning for usage of environment variables for paths containing ``$`` or ``%``
-  (`#832 <https://github.com/gitpython-developers/GitPython/issues/832>`_, 
+  (`#832 <https://github.com/gitpython-developers/GitPython/issues/832>`_,
   `#961 <https://github.com/gitpython-developers/GitPython/pull/961>`_)
 * Added support for parsing Git internal date format (@<unix timestamp> <timezone offset>)
   (`#965 <https://github.com/gitpython-developers/GitPython/pull/965>`_)
@@ -371,7 +399,7 @@ Notable fixes
 * The `GIT_DIR` environment variable does not override the `path` argument when
   initializing a `Repo` object anymore. However, if said `path` unset, `GIT_DIR`
   will be used to fill the void.
-  
+
 All issues and PRs can be viewed in all detail when following this URL:
 https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone%3A%22v2.1.0+-+proper+windows+support%22
 
@@ -401,7 +429,7 @@ https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone
 2.0.7 - New Features
 ====================
 
-* `IndexFile.commit(...,skip_hooks=False)` added. This parameter emulates the 
+* `IndexFile.commit(...,skip_hooks=False)` added. This parameter emulates the
    behaviour of `--no-verify` on the command-line.
 
 2.0.6 - Fixes and Features
@@ -441,7 +469,7 @@ https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone
   commit messages contained ``\r`` characters
 * Fix: progress handler exceptions are not caught anymore, which would usually just hide bugs
   previously.
-* Fix: The `Git.execute` method will now redirect `stdout` to `devnull` if `with_stdout` is false, 
+* Fix: The `Git.execute` method will now redirect `stdout` to `devnull` if `with_stdout` is false,
   which is the intended behaviour based on the parameter's documentation.
 
 2.0.2 - Fixes

git/cmd.py

@@ -3,7 +3,7 @@
 #
 # This module is part of GitPython and is released under
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
-
+from __future__ import annotations
 from contextlib import contextmanager
 import io
 import logging
@@ -68,7 +68,7 @@
 # Documentation
 ## @{
 
-def handle_process_output(process: Union[subprocess.Popen, 'Git.AutoInterrupt'],
+def handle_process_output(process: 'Git.AutoInterrupt' | Popen,
                           stdout_handler: Union[None,
                                                 Callable[[AnyStr], None],
                                                 Callable[[List[AnyStr]], None],
@@ -78,7 +78,8 @@ def handle_process_output(process: Union[subprocess.Popen, 'Git.AutoInterrupt'],
                                                 Callable[[List[AnyStr]], None]],
                           finalizer: Union[None,
                                            Callable[[Union[subprocess.Popen, 'Git.AutoInterrupt']], None]] = None,
-                          decode_streams: bool = True) -> None:
+                          decode_streams: bool = True,
+                          timeout: float = 10.0) -> None:
     """Registers for notifications to learn that process output is ready to read, and dispatches lines to
     the respective line handlers.
     This function returns once the finalizer returns
@@ -93,9 +94,10 @@ def handle_process_output(process: Union[subprocess.Popen, 'Git.AutoInterrupt'],
         their contents to handlers.
         Set it to False if `universal_newline == True` (then streams are in text-mode)
         or if decoding must happen later (i.e. for Diffs).
+    :param timeout: float, timeout to pass to t.join() in case it hangs. Default = 10.0 seconds
     """
     # Use 2 "pump" threads and wait for both to finish.
-    def pump_stream(cmdline: str, name: str, stream: Union[BinaryIO, TextIO], is_decode: bool,
+    def pump_stream(cmdline: List[str], name: str, stream: Union[BinaryIO, TextIO], is_decode: bool,
                     handler: Union[None, Callable[[Union[bytes, str]], None]]) -> None:
         try:
             for line in stream:
@@ -107,22 +109,32 @@ def pump_stream(cmdline: str, name: str, stream: Union[BinaryIO, TextIO], is_dec
                     else:
                         handler(line)
         except Exception as ex:
-            log.error("Pumping %r of cmd(%s) failed due to: %r", name, remove_password_if_present(cmdline), ex)
-            raise CommandError(['<%s-pump>' % name] + remove_password_if_present(cmdline), ex) from ex
+            log.error(f"Pumping {name!r} of cmd({remove_password_if_present(cmdline)}) failed due to: {ex!r}")
+            raise CommandError([f'<{name}-pump>'] + remove_password_if_present(cmdline), ex) from ex
         finally:
             stream.close()
 
-    cmdline = getattr(process, 'args', '')  # PY3+ only
+    if hasattr(process, 'proc'):
+        process = cast('Git.AutoInterrupt', process)
+        cmdline: str | Tuple[str, ...] | List[str] = getattr(process.proc, 'args', '')
+        p_stdout = process.proc.stdout if process.proc else None
+        p_stderr = process.proc.stderr if process.proc else None
+    else:
+        process = cast(Popen, process)
+        cmdline = getattr(process, 'args', '')
+        p_stdout = process.stdout
+        p_stderr = process.stderr
+
     if not isinstance(cmdline, (tuple, list)):
         cmdline = cmdline.split()
 
-    pumps = []
-    if process.stdout:
-        pumps.append(('stdout', process.stdout, stdout_handler))
-    if process.stderr:
-        pumps.append(('stderr', process.stderr, stderr_handler))
+    pumps: List[Tuple[str, IO, Callable[..., None] | None]] = []
+    if p_stdout:
+        pumps.append(('stdout', p_stdout, stdout_handler))
+    if p_stderr:
+        pumps.append(('stderr', p_stderr, stderr_handler))
 
-    threads = []
+    threads: List[threading.Thread] = []
 
     for name, stream, handler in pumps:
         t = threading.Thread(target=pump_stream,
@@ -134,7 +146,9 @@ def pump_stream(cmdline: str, name: str, stream: Union[BinaryIO, TextIO], is_dec
     ## FIXME: Why Join??  Will block if `stdin` needs feeding...
     #
     for t in threads:
-        t.join()
+        t.join(timeout=timeout)
+        if t.is_alive():
+            raise RuntimeError(f"Thread join() timed out in cmd.handle_process_output(). Timeout={timeout} seconds")
 
     if finalizer:
         return finalizer(process)

git/config.py

@@ -31,7 +31,7 @@
 # typing-------------------------------------------------------
 
 from typing import (Any, Callable, Generic, IO, List, Dict, Sequence,
-                    TYPE_CHECKING, Tuple, TypeVar, Union, cast, overload)
+                    TYPE_CHECKING, Tuple, TypeVar, Union, cast)
 
 from git.types import Lit_config_levels, ConfigLevels_Tup, PathLike, assert_never, _T
 
@@ -709,15 +709,6 @@ def read_only(self) -> bool:
         """:return: True if this instance may change the configuration file"""
         return self._read_only
 
-    @overload
-    def get_value(self, section: str, option: str, default: None = None) -> Union[int, float, str, bool]: ...
-
-    @overload
-    def get_value(self, section: str, option: str, default: str) -> str: ...
-
-    @overload
-    def get_value(self, section: str, option: str, default: float) -> float: ...
-
     def get_value(self, section: str, option: str, default: Union[int, float, str, bool, None] = None
                   ) -> Union[int, float, str, bool]:
         # can default or return type include bool?

git/objects/commit.py

@@ -446,6 +446,8 @@ def create_from_tree(cls, repo: 'Repo', tree: Union[Tree, str], message: str,
         # assume utf8 encoding
         enc_section, enc_option = cls.conf_encoding.split('.')
         conf_encoding = cr.get_value(enc_section, enc_option, cls.default_encoding)
+        if not isinstance(conf_encoding, str):
+            raise TypeError("conf_encoding could not be coerced to str")
 
         # if the tree is no object, make sure we create one - otherwise
         # the created commit object is invalid

git/objects/util.py

@@ -5,7 +5,7 @@
 # the BSD License: http://www.opensource.org/licenses/bsd-license.php
 """Module for general utility functions"""
 
-from abc import abstractmethod
+from abc import ABC, abstractmethod
 import warnings
 from git.util import (
     IterableList,
@@ -22,10 +22,10 @@
 from datetime import datetime, timedelta, tzinfo
 
 # typing ------------------------------------------------------------
-from typing import (Any, Callable, Deque, Iterator, NamedTuple, overload, Sequence,
+from typing import (Any, Callable, Deque, Iterator, Generic, NamedTuple, overload, Sequence,  # NOQA: F401
                     TYPE_CHECKING, Tuple, Type, TypeVar, Union, cast)
 
-from git.types import Has_id_attribute, Literal, Protocol, runtime_checkable
+from git.types import Has_id_attribute, Literal, _T             # NOQA: F401
 
 if TYPE_CHECKING:
     from io import BytesIO, StringIO
@@ -35,6 +35,13 @@
     from .tree import Tree, TraversedTreeTup
     from subprocess import Popen
     from .submodule.base import Submodule
+    from git.types import Protocol, runtime_checkable
+else:
+    # Protocol = Generic[_T]  # NNeeded for typing bug #572?
+    Protocol = ABC
+
+    def runtime_checkable(f):
+        return f
 
 
 class TraverseNT(NamedTuple):