This commit is contained in:
ton
2024-10-07 10:13:40 +07:00
parent aa1631742f
commit 3a7d696db6
9729 changed files with 1832837 additions and 161742 deletions

View File

@@ -9,6 +9,7 @@
__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
import errno
import io
import os
import sys
@@ -271,12 +272,22 @@ if _winapi:
with FILE_FLAG_OVERLAPPED.
"""
_got_empty_message = False
_send_ov = None
def _close(self, _CloseHandle=_winapi.CloseHandle):
ov = self._send_ov
if ov is not None:
# Interrupt WaitForMultipleObjects() in _send_bytes()
ov.cancel()
_CloseHandle(self._handle)
def _send_bytes(self, buf):
if self._send_ov is not None:
# A connection should only be used by a single thread
raise ValueError("concurrent send_bytes() calls "
"are not supported")
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
self._send_ov = ov
try:
if err == _winapi.ERROR_IO_PENDING:
waitres = _winapi.WaitForMultipleObjects(
@@ -286,7 +297,13 @@ if _winapi:
ov.cancel()
raise
finally:
self._send_ov = None
nwritten, err = ov.GetOverlappedResult(True)
if err == _winapi.ERROR_OPERATION_ABORTED:
# close() was called by another thread while
# WaitForMultipleObjects() was waiting for the overlapped
# operation.
raise OSError(errno.EPIPE, "handle is closed")
assert err == 0
assert nwritten == len(buf)
@@ -459,8 +476,9 @@ class Listener(object):
'''
if self._listener is None:
raise OSError('listener is closed')
c = self._listener.accept()
if self._authkey:
if self._authkey is not None:
deliver_challenge(c, self._authkey)
answer_challenge(c, self._authkey)
return c

View File

@@ -153,7 +153,7 @@ class Server(object):
Listener, Client = listener_client[serializer]
# do authentication later
self.listener = Listener(address=address, backlog=16)
self.listener = Listener(address=address, backlog=128)
self.address = self.listener.address
self.id_to_obj = {'0': (None, ())}

View File

@@ -14,6 +14,7 @@ __all__ = ['Popen']
#
#
# Exit code used by Popen.terminate()
TERMINATE = 0x10000
WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
@@ -100,18 +101,20 @@ class Popen(object):
return reduction.duplicate(handle, self.sentinel)
def wait(self, timeout=None):
if self.returncode is None:
if timeout is None:
msecs = _winapi.INFINITE
else:
msecs = max(0, int(timeout * 1000 + 0.5))
if self.returncode is not None:
return self.returncode
res = _winapi.WaitForSingleObject(int(self._handle), msecs)
if res == _winapi.WAIT_OBJECT_0:
code = _winapi.GetExitCodeProcess(self._handle)
if code == TERMINATE:
code = -signal.SIGTERM
self.returncode = code
if timeout is None:
msecs = _winapi.INFINITE
else:
msecs = max(0, int(timeout * 1000 + 0.5))
res = _winapi.WaitForSingleObject(int(self._handle), msecs)
if res == _winapi.WAIT_OBJECT_0:
code = _winapi.GetExitCodeProcess(self._handle)
if code == TERMINATE:
code = -signal.SIGTERM
self.returncode = code
return self.returncode
@@ -119,12 +122,22 @@ class Popen(object):
return self.wait(timeout=0)
def terminate(self):
if self.returncode is None:
try:
_winapi.TerminateProcess(int(self._handle), TERMINATE)
except OSError:
if self.wait(timeout=1.0) is None:
raise
if self.returncode is not None:
return
try:
_winapi.TerminateProcess(int(self._handle), TERMINATE)
except PermissionError:
# ERROR_ACCESS_DENIED (winerror 5) is received when the
# process already died.
code = _winapi.GetExitCodeProcess(int(self._handle))
if code == _winapi.STILL_ACTIVE:
raise
# gh-113009: Don't set self.returncode. Even if GetExitCodeProcess()
# returns an exit code different than STILL_ACTIVE, the process can
# still be running. Only set self.returncode once WaitForSingleObject()
# returns WAIT_OBJECT_0 in wait().
kill = terminate

View File

@@ -158,6 +158,20 @@ class Queue(object):
except AttributeError:
pass
def _terminate_broken(self):
# Close a Queue on error.
# gh-94777: Prevent queue writing to a pipe which is no longer read.
self._reader.close()
# gh-107219: Close the connection writer which can unblock
# Queue._feed() if it was stuck in send_bytes().
if sys.platform == 'win32':
self._writer.close()
self.close()
self.join_thread()
def _start_thread(self):
debug('Queue._start_thread()')
@@ -169,13 +183,19 @@ class Queue(object):
self._wlock, self._reader.close, self._writer.close,
self._ignore_epipe, self._on_queue_feeder_error,
self._sem),
name='QueueFeederThread'
name='QueueFeederThread',
daemon=True,
)
self._thread.daemon = True
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
try:
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
except:
# gh-109047: During Python finalization, creating a thread
# can fail with RuntimeError.
self._thread = None
raise
if not self._joincancelled:
self._jointhread = Finalize(

View File

@@ -123,7 +123,7 @@ class _ResourceSharer(object):
from .connection import Listener
assert self._listener is None, "Already have Listener"
util.debug('starting listener and thread for sending handles')
self._listener = Listener(authkey=process.current_process().authkey)
self._listener = Listener(authkey=process.current_process().authkey, backlog=128)
self._address = self._listener.address
t = threading.Thread(target=self._serve)
t.daemon = True

View File

@@ -51,15 +51,31 @@ if os.name == 'posix':
})
class ReentrantCallError(RuntimeError):
pass
class ResourceTracker(object):
def __init__(self):
self._lock = threading.Lock()
self._lock = threading.RLock()
self._fd = None
self._pid = None
def _reentrant_call_error(self):
# gh-109629: this happens if an explicit call to the ResourceTracker
# gets interrupted by a garbage collection, invoking a finalizer (*)
# that itself calls back into ResourceTracker.
# (*) for example the SemLock finalizer
raise ReentrantCallError(
"Reentrant call into the multiprocessing resource tracker")
def _stop(self):
with self._lock:
# This should not happen (_stop() isn't called by a finalizer)
# but we check for it anyway.
if self._lock._recursion_count() > 1:
return self._reentrant_call_error()
if self._fd is None:
# not running
return
@@ -81,6 +97,9 @@ class ResourceTracker(object):
This can be run from any process. Usually a child process will use
the resource created by its parent.'''
with self._lock:
if self._lock._recursion_count() > 1:
# The code below is certainly not reentrant-safe, so bail out
return self._reentrant_call_error()
if self._fd is not None:
# resource tracker was launched before, is it still running?
if self._check_alive():
@@ -159,7 +178,17 @@ class ResourceTracker(object):
self._send('UNREGISTER', name, rtype)
def _send(self, cmd, name, rtype):
self.ensure_running()
try:
self.ensure_running()
except ReentrantCallError:
# The code below might or might not work, depending on whether
# the resource tracker was already running and still alive.
# Better warn the user.
# (XXX is warnings.warn itself reentrant-safe? :-)
warnings.warn(
f"ResourceTracker called reentrantly for resource cleanup, "
f"which is unsupported. "
f"The {rtype} object {name!r} might leak.")
msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii')
if len(msg) > 512:
# posix guarantees that writes to a pipe of less than PIPE_BUF
@@ -176,6 +205,7 @@ register = _resource_tracker.register
unregister = _resource_tracker.unregister
getfd = _resource_tracker.getfd
def main(fd):
'''Run resource tracker.'''
# protect the process from ^C and "killall python" etc

View File

@@ -43,19 +43,19 @@ _log_to_stderr = False
def sub_debug(msg, *args):
if _logger:
_logger.log(SUBDEBUG, msg, *args)
_logger.log(SUBDEBUG, msg, *args, stacklevel=2)
def debug(msg, *args):
if _logger:
_logger.log(DEBUG, msg, *args)
_logger.log(DEBUG, msg, *args, stacklevel=2)
def info(msg, *args):
if _logger:
_logger.log(INFO, msg, *args)
_logger.log(INFO, msg, *args, stacklevel=2)
def sub_warning(msg, *args):
if _logger:
_logger.log(SUBWARNING, msg, *args)
_logger.log(SUBWARNING, msg, *args, stacklevel=2)
def get_logger():
'''
@@ -130,7 +130,10 @@ abstract_sockets_supported = _platform_supports_abstract_sockets()
#
def _remove_temp_dir(rmtree, tempdir):
rmtree(tempdir)
def onerror(func, path, err_info):
if not issubclass(err_info[0], FileNotFoundError):
raise
rmtree(tempdir, onerror=onerror)
current_process = process.current_process()
# current_process() can be None if the finalizer is called