This commit is contained in:
ton
2024-10-07 10:13:40 +07:00
parent aa1631742f
commit 3a7d696db6
9729 changed files with 1832837 additions and 161742 deletions

View File

@@ -16,7 +16,6 @@ class AsyncIOInteractiveConsole(code.InteractiveConsole):
def __init__(self, locals, loop):
super().__init__(locals)
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
self.loop = loop
def runcode(self, code):
@@ -90,6 +89,8 @@ class REPLThread(threading.Thread):
if __name__ == '__main__':
sys.audit("cpython.run_stdin")
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)

View File

@@ -16,6 +16,7 @@ to modify the meaning of the API call itself.
import collections
import collections.abc
import concurrent.futures
import errno
import functools
import heapq
import itertools
@@ -44,6 +45,7 @@ from . import protocols
from . import sslproto
from . import staggered
from . import tasks
from . import timeouts
from . import transports
from . import trsock
from .log import logger
@@ -305,7 +307,7 @@ class Server(events.AbstractServer):
self._waiters = None
for waiter in waiters:
if not waiter.done():
waiter.set_result(waiter)
waiter.set_result(None)
def _start_serving(self):
if self._serving:
@@ -377,7 +379,27 @@ class Server(events.AbstractServer):
self._serving_forever_fut = None
async def wait_closed(self):
if self._waiters is None or self._active_count == 0:
"""Wait until server is closed and all connections are dropped.
- If the server is not closed, wait.
- If it is closed, but there are still active connections, wait.
Anyone waiting here will be unblocked once both conditions
(server is closed and all connections have been dropped)
have become true, in either order.
Historical note: In 3.11 and before, this was broken, returning
immediately if the server was already closed, even if there
were still active connections. An attempted fix in 3.12.0 was
still broken, returning immediately if the server was still
open and there were no active connections. Hopefully in 3.12.1
we have it right.
"""
# Waiters are unblocked by self._wakeup(), which is called
# from two places: self.close() and self._detach(), but only
# when both conditions have become true. To signal that this
# has happened, self._wakeup() sets self._waiters to None.
if self._waiters is None:
return
waiter = self._loop.create_future()
self._waiters.append(waiter)
@@ -575,23 +597,24 @@ class BaseEventLoop(events.AbstractEventLoop):
thread = threading.Thread(target=self._do_shutdown, args=(future,))
thread.start()
try:
await future
finally:
thread.join(timeout)
if thread.is_alive():
async with timeouts.timeout(timeout):
await future
except TimeoutError:
warnings.warn("The executor did not finishing joining "
f"its threads within {timeout} seconds.",
RuntimeWarning, stacklevel=2)
f"its threads within {timeout} seconds.",
RuntimeWarning, stacklevel=2)
self._default_executor.shutdown(wait=False)
else:
thread.join()
def _do_shutdown(self, future):
try:
self._default_executor.shutdown(wait=True)
if not self.is_closed():
self.call_soon_threadsafe(future.set_result, None)
self.call_soon_threadsafe(futures._set_result_unless_cancelled,
future, None)
except Exception as ex:
if not self.is_closed():
if not self.is_closed() and not future.cancelled():
self.call_soon_threadsafe(future.set_exception, ex)
def _check_running(self):
@@ -971,8 +994,7 @@ class BaseEventLoop(events.AbstractEventLoop):
except OSError as exc:
msg = (
f'error while attempting to bind on '
f'address {laddr!r}: '
f'{exc.strerror.lower()}'
f'address {laddr!r}: {str(exc).lower()}'
)
exc = OSError(exc.errno, msg)
my_exceptions.append(exc)
@@ -1294,9 +1316,9 @@ class BaseEventLoop(events.AbstractEventLoop):
allow_broadcast=None, sock=None):
"""Create datagram connection."""
if sock is not None:
if sock.type != socket.SOCK_DGRAM:
if sock.type == socket.SOCK_STREAM:
raise ValueError(
f'A UDP Socket was expected, got {sock!r}')
f'A datagram socket was expected, got {sock!r}')
if (local_addr or remote_addr or
family or proto or flags or
reuse_port or allow_broadcast):
@@ -1536,9 +1558,22 @@ class BaseEventLoop(events.AbstractEventLoop):
try:
sock.bind(sa)
except OSError as err:
raise OSError(err.errno, 'error while attempting '
'to bind on address %r: %s'
% (sa, err.strerror.lower())) from None
msg = ('error while attempting '
'to bind on address %r: %s'
% (sa, str(err).lower()))
if err.errno == errno.EADDRNOTAVAIL:
# Assume the family is not enabled (bpo-30945)
sockets.pop()
sock.close()
if self._debug:
logger.warning(msg)
continue
raise OSError(err.errno, msg) from None
if not sockets:
raise OSError('could not bind on any address out of %r'
% ([info[4] for info in infos],))
completed = True
finally:
if not completed:

View File

@@ -1,3 +1,7 @@
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
import enum
# After the connection is lost, log warnings after this many write()s.

View File

@@ -1,5 +1,9 @@
"""Event loop and event loop policy."""
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
__all__ = (
'AbstractEventLoopPolicy',
'AbstractEventLoop', 'AbstractServer',

View File

@@ -272,9 +272,13 @@ class Future:
raise exceptions.InvalidStateError(f'{self._state}: {self!r}')
if isinstance(exception, type):
exception = exception()
if type(exception) is StopIteration:
raise TypeError("StopIteration interacts badly with generators "
"and cannot be raised into a Future")
if isinstance(exception, StopIteration):
new_exc = RuntimeError("StopIteration interacts badly with "
"generators and cannot be raised into a "
"Future")
new_exc.__cause__ = exception
new_exc.__context__ = exception
exception = new_exc
self._exception = exception
self._exception_tb = exception.__traceback__
self._state = _FINISHED

View File

@@ -724,6 +724,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop):
return await self._proactor.sendto(sock, data, 0, address)
async def sock_connect(self, sock, address):
if self._debug and sock.gettimeout() != 0:
raise ValueError("the socket must be non-blocking")
return await self._proactor.connect(sock, address)
async def sock_accept(self, sock):

View File

@@ -235,6 +235,10 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop):
await waiter
except BaseException:
transport.close()
# gh-109534: When an exception is raised by the SSLProtocol object the
# exception set in this future can keep the protocol object alive and
# cause a reference cycle.
waiter = None
raise
# It's now up to the protocol to handle the connection.

View File

@@ -1,3 +1,7 @@
# Contains code from https://github.com/MagicStack/uvloop/tree/v0.16.0
# SPDX-License-Identifier: PSF-2.0 AND (MIT OR Apache-2.0)
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
import collections
import enum
import warnings
@@ -243,13 +247,12 @@ class _SSLProtocolTransport(transports._FlowControlMixin,
The protocol's connection_lost() method will (eventually) be
called with None as its argument.
"""
self._closed = True
if self._ssl_protocol is not None:
self._ssl_protocol._abort()
self._force_close(None)
def _force_close(self, exc):
self._closed = True
self._ssl_protocol._abort(exc)
if self._ssl_protocol is not None:
self._ssl_protocol._abort(exc)
def _test__append_write_backlog(self, data):
# for test only
@@ -576,6 +579,7 @@ class SSLProtocol(protocols.BufferedProtocol):
peercert = sslobj.getpeercert()
except Exception as exc:
handshake_exc = None
self._set_state(SSLProtocolState.UNWRAPPED)
if isinstance(exc, ssl.CertificateError):
msg = 'SSL handshake failed on verifying the certificate'
@@ -614,7 +618,7 @@ class SSLProtocol(protocols.BufferedProtocol):
if self._app_transport is not None:
self._app_transport._closed = True
if self._state == SSLProtocolState.DO_HANDSHAKE:
self._abort()
self._abort(None)
else:
self._set_state(SSLProtocolState.FLUSHING)
self._shutdown_timeout_handle = self._loop.call_later(
@@ -661,10 +665,10 @@ class SSLProtocol(protocols.BufferedProtocol):
else:
self._loop.call_soon(self._transport.close)
def _abort(self):
def _abort(self, exc):
self._set_state(SSLProtocolState.UNWRAPPED)
if self._transport is not None:
self._transport.abort()
self._transport._force_close(exc)
# Outgoing flow

View File

@@ -3,7 +3,6 @@
__all__ = 'staggered_race',
import contextlib
import typing
from . import events
from . import exceptions as exceptions_mod
@@ -11,16 +10,7 @@ from . import locks
from . import tasks
async def staggered_race(
coro_fns: typing.Iterable[typing.Callable[[], typing.Awaitable]],
delay: typing.Optional[float],
*,
loop: events.AbstractEventLoop = None,
) -> typing.Tuple[
typing.Any,
typing.Optional[int],
typing.List[typing.Optional[Exception]]
]:
async def staggered_race(coro_fns, delay, *, loop=None):
"""Run coroutines with staggered start times and take the first to finish.
This method takes an iterable of coroutine functions. The first one is
@@ -79,8 +69,7 @@ async def staggered_race(
exceptions = []
running_tasks = []
async def run_one_coro(
previous_failed: typing.Optional[locks.Event]) -> None:
async def run_one_coro(previous_failed) -> None:
# Wait for the previous task to finish, or for delay seconds
if previous_failed is not None:
with contextlib.suppress(exceptions_mod.TimeoutError):

View File

@@ -5,6 +5,7 @@ __all__ = (
import collections
import socket
import sys
import warnings
import weakref
if hasattr(socket, 'AF_UNIX'):
@@ -66,9 +67,8 @@ async def start_server(client_connected_cb, host=None, port=None, *,
positional host and port, with various optional keyword arguments
following. The return value is the same as loop.create_server().
Additional optional keyword arguments are loop (to set the event loop
instance to use) and limit (to set the buffer limit passed to the
StreamReader).
Additional optional keyword argument is limit (to set the buffer
limit passed to the StreamReader).
The return value is the same as loop.create_server(), i.e. a
Server object which can be used to stop the service.
@@ -245,7 +245,22 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
res = self._client_connected_cb(reader,
self._stream_writer)
if coroutines.iscoroutine(res):
def callback(task):
if task.cancelled():
transport.close()
return
exc = task.exception()
if exc is not None:
self._loop.call_exception_handler({
'message': 'Unhandled exception in client_connected_cb',
'exception': exc,
'transport': transport,
})
transport.close()
self._task = self._loop.create_task(res)
self._task.add_done_callback(callback)
self._strong_reader = None
def connection_lost(self, exc):
@@ -394,8 +409,11 @@ class StreamWriter:
def __del__(self):
if not self._transport.is_closing():
self.close()
if self._loop.is_closed():
warnings.warn("loop is closed", ResourceWarning)
else:
self.close()
warnings.warn(f"unclosed {self!r}", ResourceWarning)
class StreamReader:

View File

@@ -147,15 +147,17 @@ class Process:
async def _feed_stdin(self, input):
debug = self._loop.get_debug()
if input is not None:
self.stdin.write(input)
if debug:
logger.debug(
'%r communicate: feed stdin (%s bytes)', self, len(input))
try:
if input is not None:
self.stdin.write(input)
if debug:
logger.debug(
'%r communicate: feed stdin (%s bytes)', self, len(input))
await self.stdin.drain()
except (BrokenPipeError, ConnectionResetError) as exc:
# communicate() ignores BrokenPipeError and ConnectionResetError
# communicate() ignores BrokenPipeError and ConnectionResetError.
# write() and drain() can raise these exceptions.
if debug:
logger.debug('%r communicate: stdin got %r', self, exc)

View File

@@ -54,16 +54,14 @@ class TaskGroup:
async def __aenter__(self):
if self._entered:
raise RuntimeError(
f"TaskGroup {self!r} has been already entered")
self._entered = True
f"TaskGroup {self!r} has already been entered")
if self._loop is None:
self._loop = events.get_running_loop()
self._parent_task = tasks.current_task(self._loop)
if self._parent_task is None:
raise RuntimeError(
f'TaskGroup {self!r} cannot determine the parent task')
self._entered = True
return self

View File

@@ -86,15 +86,25 @@ class Task(futures._PyFuture): # Inherit Python Task implementation
"""A coroutine wrapped in a Future."""
# An important invariant maintained while a Task not done:
# _fut_waiter is either None or a Future. The Future
# can be either done() or not done().
# The task can be in any of 3 states:
#
# - Either _fut_waiter is None, and _step() is scheduled;
# - or _fut_waiter is some Future, and _step() is *not* scheduled.
# - 1: _fut_waiter is not None and not _fut_waiter.done():
# __step() is *not* scheduled and the Task is waiting for _fut_waiter.
# - 2: (_fut_waiter is None or _fut_waiter.done()) and __step() is scheduled:
# the Task is waiting for __step() to be executed.
# - 3: _fut_waiter is None and __step() is *not* scheduled:
# the Task is currently executing (in __step()).
#
# The only transition from the latter to the former is through
# _wakeup(). When _fut_waiter is not None, one of its callbacks
# must be _wakeup().
# * In state 1, one of the callbacks of __fut_waiter must be __wakeup().
# * The transition from 1 to 2 happens when _fut_waiter becomes done(),
# as it schedules __wakeup() to be called (which calls __step() so
# we way that __step() is scheduled).
# * It transitions from 2 to 3 when __step() is executed, and it clears
# _fut_waiter to None.
# If False, don't log a message if the task is destroyed whereas its
# If False, don't log a message if the task is destroyed while its
# status is still pending
_log_destroy_pending = True
@@ -470,7 +480,7 @@ async def wait_for(fut, timeout):
If the wait is cancelled, the task is also cancelled.
If the task supresses the cancellation and returns a value instead,
If the task suppresses the cancellation and returns a value instead,
that value is returned.
This function is a coroutine.

View File

@@ -49,8 +49,9 @@ class Timeout:
def reschedule(self, when: Optional[float]) -> None:
"""Reschedule the timeout."""
assert self._state is not _State.CREATED
if self._state is not _State.ENTERED:
if self._state is _State.CREATED:
raise RuntimeError("Timeout has not been entered")
raise RuntimeError(
f"Cannot change state of {self._state.value} Timeout",
)
@@ -82,11 +83,14 @@ class Timeout:
return f"<Timeout [{self._state.value}]{info_str}>"
async def __aenter__(self) -> "Timeout":
self._state = _State.ENTERED
self._task = tasks.current_task()
self._cancelling = self._task.cancelling()
if self._task is None:
if self._state is not _State.CREATED:
raise RuntimeError("Timeout has already been entered")
task = tasks.current_task()
if task is None:
raise RuntimeError("Timeout should be used inside a task")
self._state = _State.ENTERED
self._task = task
self._cancelling = self._task.cancelling()
self.reschedule(self._when)
return self

View File

@@ -226,8 +226,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
return transp
def _child_watcher_callback(self, pid, returncode, transp):
# Skip one iteration for callbacks to be executed
self.call_soon_threadsafe(self.call_soon, transp._process_exited, returncode)
self.call_soon_threadsafe(transp._process_exited, returncode)
async def create_unix_connection(
self, protocol_factory, path=None, *,
@@ -1368,14 +1367,7 @@ class ThreadedChildWatcher(AbstractChildWatcher):
return True
def close(self):
self._join_threads()
def _join_threads(self):
"""Internal: Join all non-daemon threads"""
threads = [thread for thread in list(self._threads.values())
if thread.is_alive() and not thread.daemon]
for thread in threads:
thread.join()
pass
def __enter__(self):
return self
@@ -1394,7 +1386,7 @@ class ThreadedChildWatcher(AbstractChildWatcher):
def add_child_handler(self, pid, callback, *args):
loop = events.get_running_loop()
thread = threading.Thread(target=self._do_waitpid,
name=f"waitpid-{next(self._pid_counter)}",
name=f"asyncio-waitpid-{next(self._pid_counter)}",
args=(loop, pid, callback, args),
daemon=True)
self._threads[pid] = thread
@@ -1462,8 +1454,6 @@ class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
self._watcher = PidfdChildWatcher()
else:
self._watcher = ThreadedChildWatcher()
if threading.current_thread() is threading.main_thread():
self._watcher.attach_loop(self._local._loop)
def set_event_loop(self, loop):
"""Set the event loop.

View File

@@ -8,6 +8,7 @@ if sys.platform != 'win32': # pragma: no cover
import _overlapped
import _winapi
import errno
from functools import partial
import math
import msvcrt
import socket
@@ -323,13 +324,13 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
if self._self_reading_future is not None:
ov = self._self_reading_future._ov
self._self_reading_future.cancel()
# self_reading_future was just cancelled so if it hasn't been
# finished yet, it never will be (it's possible that it has
# already finished and its callback is waiting in the queue,
# where it could still happen if the event loop is restarted).
# Unregister it otherwise IocpProactor.close will wait for it
# forever
if ov is not None:
# self_reading_future always uses IOCP, so even though it's
# been cancelled, we need to make sure that the IOCP message
# is received so that the kernel is not holding on to the
# memory, possibly causing memory corruption later. Only
# unregister it if IO is complete in all respects. Otherwise
# we need another _poll() later to complete the IO.
if ov is not None and not ov.pending:
self._proactor._unregister(ov)
self._self_reading_future = None
@@ -466,6 +467,18 @@ class IocpProactor:
else:
raise
@classmethod
def _finish_recvfrom(cls, trans, key, ov, *, empty_result):
try:
return cls.finish_socket_func(trans, key, ov)
except OSError as exc:
# WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same
# socket is used to send to an address that is not listening.
if exc.winerror == _overlapped.ERROR_PORT_UNREACHABLE:
return empty_result, None
else:
raise
def recv(self, conn, nbytes, flags=0):
self._register_with_iocp(conn)
ov = _overlapped.Overlapped(NULL)
@@ -500,7 +513,8 @@ class IocpProactor:
except BrokenPipeError:
return self._result((b'', None))
return self._register(ov, conn, self.finish_socket_func)
return self._register(ov, conn, partial(self._finish_recvfrom,
empty_result=b''))
def recvfrom_into(self, conn, buf, flags=0):
self._register_with_iocp(conn)
@@ -510,17 +524,8 @@ class IocpProactor:
except BrokenPipeError:
return self._result((0, None))
def finish_recv(trans, key, ov):
try:
return ov.getresult()
except OSError as exc:
if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED,
_overlapped.ERROR_OPERATION_ABORTED):
raise ConnectionResetError(*exc.args)
else:
raise
return self._register(ov, conn, finish_recv)
return self._register(ov, conn, partial(self._finish_recvfrom,
empty_result=0))
def sendto(self, conn, buf, flags=0, addr=None):
self._register_with_iocp(conn)