[BUG] 60.7.0 breaks gevent monkeypatching #3090
Description
setuptools version
setuptools>=60.7.0
Python version
Python 3.x (tested on 3.9 & 3.10)
OS
Any (tested on Linux and MacOS)
Additional environment information
No response
Description
Recent changes in Setuptools, introduced in version 60.7.0, break gevent concurrent networking library. When setuptools>=60.7.0 is installed, gevent is unable to monkey-patch the standard library and freezes in runtime.
There are more detail in gevent/gevent#1865 and the script used in reproduction steps comes from that issue.
Gevent monkey-patches networking and concurrency functions in standard library to replace them with versions that allow other green threads to run when the function is waiting for I/O. This monkey-patching needs to happen before any of the patched modules are loaded.
One of the changes in Setuptools 60.7.0 was using a vendored more_itertools library. This library, in turn, imports concurrent.futures.threading, which is one of the modules monkey-patched by gevent. Gevent code includes a from pkg_resources import iter_entry_points line, which triggers more_itertools import when gevent is imported, which creates a chicken-and-egg problem.
This is fixed by a following quick-and-dirty patch to setuptools, from which I will prepare a proper pull request. I'm sending a bug report first to provide context, because it will require a change in jaraco.text too. Note that this is a proof of concept, not a final proposed change.
git diff
diff --git i/pkg_resources/_vendor/jaraco/functools.py w/pkg_resources/_vendor/jaraco/functools.py index a3fea3a1..e8f47eb8 100644 --- i/pkg_resources/_vendor/jaraco/functools.py +++ w/pkg_resources/_vendor/jaraco/functools.py @@ -5,14 +5,18 @@ import collections import types import itertools -import pkg_resources.extern.more_itertools - from typing import Callable, TypeVar CallableT = TypeVar("CallableT", bound=Callable[..., object]) +def _consume(iterator): + """Consume iterator (itertools recipe)""" + # feed the entire iterator into a zero-length deque + collections.deque(iterator, maxlen=0) + + def compose(*funcs): """ Compose any number of unary functions into a single unary function. @@ -385,7 +389,7 @@ def print_yielded(func): None """ print_all = functools.partial(map, print) - print_results = compose(more_itertools.consume, print_all, func) + print_results = compose(_consume, print_all, func) return functools.wraps(func)(print_results) diff --git i/setuptools/_vendor/jaraco/functools.py w/setuptools/_vendor/jaraco/functools.py index bbd8b29f..e8f47eb8 100644 --- i/setuptools/_vendor/jaraco/functools.py +++ w/setuptools/_vendor/jaraco/functools.py @@ -5,14 +5,18 @@ import collections import types import itertools -import setuptools.extern.more_itertools - from typing import Callable, TypeVar CallableT = TypeVar("CallableT", bound=Callable[..., object]) +def _consume(iterator): + """Consume iterator (itertools recipe)""" + # feed the entire iterator into a zero-length deque + collections.deque(iterator, maxlen=0) + + def compose(*funcs): """ Compose any number of unary functions into a single unary function. @@ -385,7 +389,7 @@ def print_yielded(func): None """ print_all = functools.partial(map, print) - print_results = compose(more_itertools.consume, print_all, func) + print_results = compose(_consume, print_all, func) return functools.wraps(func)(print_results) diff --git i/setuptools/command/build_py.py w/setuptools/command/build_py.py index c3fdc092..a946d1fd 100644 --- i/setuptools/command/build_py.py +++ w/setuptools/command/build_py.py @@ -8,7 +8,7 @@ import io import distutils.errors import itertools import stat -from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.more_itertools.recipes import unique_everseen def make_writable(target): diff --git i/setuptools/command/test.py w/setuptools/command/test.py index 4a389e4d..17bc69ee 100644 --- i/setuptools/command/test.py +++ w/setuptools/command/test.py @@ -19,7 +19,7 @@ from pkg_resources import ( EntryPoint, ) from setuptools import Command -from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.more_itertools.recipes import unique_everseen class ScanningLoader(TestLoader): diff --git i/setuptools/dist.py w/setuptools/dist.py index 733ae14f..ef00e417 100644 --- i/setuptools/dist.py +++ w/setuptools/dist.py @@ -28,7 +28,7 @@ from distutils.util import rfc822_escape from setuptools.extern import packaging from setuptools.extern import ordered_set -from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.more_itertools.recipes import unique_everseen from . import SetuptoolsDeprecationWarning diff --git i/setuptools/msvc.py w/setuptools/msvc.py index 281ea1c2..5908a06e 100644 --- i/setuptools/msvc.py +++ w/setuptools/msvc.py @@ -30,7 +30,7 @@ import itertools import subprocess import distutils.errors from setuptools.extern.packaging.version import LegacyVersion -from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.more_itertools.recipes import unique_everseen from .monkey import get_unpatched diff --git i/setuptools/package_index.py w/setuptools/package_index.py index 051e523a..f85a4b5e 100644 --- i/setuptools/package_index.py +++ w/setuptools/package_index.py @@ -27,7 +27,7 @@ from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.wheel import Wheel -from setuptools.extern.more_itertools import unique_everseen +from setuptools.extern.more_itertools.recipes import unique_everseen EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')unique_everseenis imported directly from therecipessubmodule which doesn't importconcurrency. I'm not sure it's necessary or that it doesn't importmore_itertools.morevia parent module anyway. This function is 16 lines copied from stdlib documentation, so in final PR I'll probably consider vendoring just this one functionconsumeis one-liner from stdlib documentation (only this branch of upstream implementation'sifis used), so it was pasted directly into source- The
more_itertoolslibrary is the largest part of 60.7.0 diff. It's about 5KLOC, vendored twice (insetuptoolsand inpkg_resources). Out of this amount of code, only two functions are used; one is a one-linerconsumeand the other is 16 lines of code.
The freezes can and should be fixed in gevent (by using importlib.metadata / importlib_metadata on Python 3; if I'm right, this version of Setuptools is Python 3 only, so pkg_resources on Python 2.7 should be safe), but I believe that at the same time it requires a fix on Setuptools side, for the following reasons:
- Minimize side effects of simply importing a library as low-level and fundamental as
setuptools/pkg_resources - Reduce size of codebase by removing second largest vendored dependency
more_itertools(or postprocessing it intools/vendored.pyto remove the problematicmore_itertools.moreand keeping onlymore_itertools.recipes)
Expected behavior
I expected that the test code from "How to Reproduce" section will run.
How to Reproduce
- Install
geventandsetuptools>=60.7.0 - Run the following code
from gevent import monkey import gevent if __name__ == "__main__": monkey.patch_thread() monkey.patch_subprocess() import subprocess from concurrent.futures import ThreadPoolExecutor def func(): # gevent.sleep(0) # uncomment to fail return subprocess.run(["ls", "-l", "/dev/null"], capture_output=True) with ThreadPoolExecutor(max_workers=5) as executor: fut = executor.submit(func) print(fut.result())The script will freeze.
Output
% python3 -m venv ./venv % . ./venv/bin/activate (venv) % pip install -U pip gevent setuptools==60.6.0 Requirement already satisfied: pip in ./venv/lib/python3.9/site-packages (21.3.1) Collecting pip Using cached pip-22.0.3-py3-none-any.whl (2.1 MB) Collecting gevent Using cached gevent-21.12.0-cp39-cp39-macosx_12_0_arm64.whl Collecting setuptools==60.6.0 Using cached setuptools-60.6.0-py3-none-any.whl (953 kB) Collecting greenlet<2.0,>=1.1.0 Using cached greenlet-1.1.2-cp39-cp39-macosx_12_0_arm64.whl Collecting zope.interface Using cached zope.interface-5.4.0-cp39-cp39-macosx_12_0_arm64.whl Collecting zope.event Using cached zope.event-4.5.0-py2.py3-none-any.whl (6.8 kB) Installing collected packages: setuptools, zope.interface, zope.event, greenlet, pip, gevent Attempting uninstall: setuptools Found existing installation: setuptools 60.5.0 Uninstalling setuptools-60.5.0: Successfully uninstalled setuptools-60.5.0 Attempting uninstall: pip Found existing installation: pip 21.3.1 Uninstalling pip-21.3.1: Successfully uninstalled pip-21.3.1 Successfully installed gevent-21.12.0 greenlet-1.1.2 pip-22.0.3 setuptools-60.6.0 zope.event-4.5.0 zope.interface-5.4.0 (venv) % cat test.py from gevent import monkey import gevent if __name__ == "__main__": monkey.patch_thread() monkey.patch_subprocess() import subprocess from concurrent.futures import ThreadPoolExecutor def func(): # gevent.sleep(0) # uncomment to fail return subprocess.run(["ls", "-l", "/dev/null"], capture_output=True) with ThreadPoolExecutor(max_workers=5) as executor: fut = executor.submit(func) print(fut.result()) (venv) % python test.py CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, stdout=b'crw-rw-rw- 1 root wheel 0x3000002 Feb 8 17:38 /dev/null\n', stderr=b'') (venv) % pip install -U setuptools Requirement already satisfied: setuptools in ./venv/lib/python3.9/site-packages (60.6.0) Collecting setuptools Using cached setuptools-60.8.1-py3-none-any.whl (1.1 MB) Installing collected packages: setuptools Attempting uninstall: setuptools Found existing installation: setuptools 60.6.0 Uninstalling setuptools-60.6.0: Successfully uninstalled setuptools-60.6.0 Successfully installed setuptools-60.8.1 (venv) % python test.py # Script freezes, interrupted with Ctrl+C ^CException ignored in: <built-in method acquire of _thread.lock object at 0x1037e26f0> Traceback (most recent call last): File "/Users/maciejp/Src/github.com/HealthByRo/tmp/testink/venv/lib/python3.9/site-packages/gevent/os.py", line 431, in fork_and_watch pid = fork() KeyboardInterrupt: Traceback (most recent call last): File "/Users/maciejp/Src/github.com/HealthByRo/tmp/testink/test.py", line 16, in <module> fut = executor.submit(func) File "/opt/homebrew/Cellar/python@3.9/3.9.10/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py", line 177, in submit return f RuntimeError: release unlocked lock (venv) %