547 lines
20 KiB
Python
547 lines
20 KiB
Python
"""Event loop using a proactor and related classes.
|
|
|
|
A proactor is a "notify-on-completion" multiplexer. Currently a
|
|
proactor is only implemented on Windows with IOCP.
|
|
"""
|
|
|
|
__all__ = ['BaseProactorEventLoop']
|
|
|
|
import socket
|
|
import warnings
|
|
|
|
from . import base_events
|
|
from . import compat
|
|
from . import constants
|
|
from . import futures
|
|
from . import sslproto
|
|
from . import transports
|
|
from .log import logger
|
|
|
|
|
|
class _ProactorBasePipeTransport(transports._FlowControlMixin,
|
|
transports.BaseTransport):
|
|
"""Base class for pipe and socket transports."""
|
|
|
|
def __init__(self, loop, sock, protocol, waiter=None,
|
|
extra=None, server=None):
|
|
super().__init__(extra, loop)
|
|
self._set_extra(sock)
|
|
self._sock = sock
|
|
self._protocol = protocol
|
|
self._server = server
|
|
self._buffer = None # None or bytearray.
|
|
self._read_fut = None
|
|
self._write_fut = None
|
|
self._pending_write = 0
|
|
self._conn_lost = 0
|
|
self._closing = False # Set when close() called.
|
|
self._eof_written = False
|
|
if self._server is not None:
|
|
self._server._attach()
|
|
self._loop.call_soon(self._protocol.connection_made, self)
|
|
if waiter is not None:
|
|
# only wake up the waiter when connection_made() has been called
|
|
self._loop.call_soon(waiter._set_result_unless_cancelled, None)
|
|
|
|
def __repr__(self):
|
|
info = [self.__class__.__name__]
|
|
if self._sock is None:
|
|
info.append('closed')
|
|
elif self._closing:
|
|
info.append('closing')
|
|
if self._sock is not None:
|
|
info.append('fd=%s' % self._sock.fileno())
|
|
if self._read_fut is not None:
|
|
info.append('read=%s' % self._read_fut)
|
|
if self._write_fut is not None:
|
|
info.append("write=%r" % self._write_fut)
|
|
if self._buffer:
|
|
bufsize = len(self._buffer)
|
|
info.append('write_bufsize=%s' % bufsize)
|
|
if self._eof_written:
|
|
info.append('EOF written')
|
|
return '<%s>' % ' '.join(info)
|
|
|
|
def _set_extra(self, sock):
|
|
self._extra['pipe'] = sock
|
|
|
|
def close(self):
|
|
if self._closing:
|
|
return
|
|
self._closing = True
|
|
self._conn_lost += 1
|
|
if not self._buffer and self._write_fut is None:
|
|
self._loop.call_soon(self._call_connection_lost, None)
|
|
if self._read_fut is not None:
|
|
self._read_fut.cancel()
|
|
self._read_fut = None
|
|
|
|
# On Python 3.3 and older, objects with a destructor part of a reference
|
|
# cycle are never destroyed. It's not more the case on Python 3.4 thanks
|
|
# to the PEP 442.
|
|
if compat.PY34:
|
|
def __del__(self):
|
|
if self._sock is not None:
|
|
warnings.warn("unclosed transport %r" % self, ResourceWarning)
|
|
self.close()
|
|
|
|
def _fatal_error(self, exc, message='Fatal error on pipe transport'):
|
|
if isinstance(exc, (BrokenPipeError, ConnectionResetError)):
|
|
if self._loop.get_debug():
|
|
logger.debug("%r: %s", self, message, exc_info=True)
|
|
else:
|
|
self._loop.call_exception_handler({
|
|
'message': message,
|
|
'exception': exc,
|
|
'transport': self,
|
|
'protocol': self._protocol,
|
|
})
|
|
self._force_close(exc)
|
|
|
|
def _force_close(self, exc):
|
|
if self._closing:
|
|
return
|
|
self._closing = True
|
|
self._conn_lost += 1
|
|
if self._write_fut:
|
|
self._write_fut.cancel()
|
|
self._write_fut = None
|
|
if self._read_fut:
|
|
self._read_fut.cancel()
|
|
self._read_fut = None
|
|
self._pending_write = 0
|
|
self._buffer = None
|
|
self._loop.call_soon(self._call_connection_lost, exc)
|
|
|
|
def _call_connection_lost(self, exc):
|
|
try:
|
|
self._protocol.connection_lost(exc)
|
|
finally:
|
|
# XXX If there is a pending overlapped read on the other
|
|
# end then it may fail with ERROR_NETNAME_DELETED if we
|
|
# just close our end. First calling shutdown() seems to
|
|
# cure it, but maybe using DisconnectEx() would be better.
|
|
if hasattr(self._sock, 'shutdown'):
|
|
self._sock.shutdown(socket.SHUT_RDWR)
|
|
self._sock.close()
|
|
self._sock = None
|
|
server = self._server
|
|
if server is not None:
|
|
server._detach()
|
|
self._server = None
|
|
|
|
def get_write_buffer_size(self):
|
|
size = self._pending_write
|
|
if self._buffer is not None:
|
|
size += len(self._buffer)
|
|
return size
|
|
|
|
|
|
class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
|
|
transports.ReadTransport):
|
|
"""Transport for read pipes."""
|
|
|
|
def __init__(self, loop, sock, protocol, waiter=None,
|
|
extra=None, server=None):
|
|
super().__init__(loop, sock, protocol, waiter, extra, server)
|
|
self._paused = False
|
|
self._loop.call_soon(self._loop_reading)
|
|
|
|
def pause_reading(self):
|
|
if self._closing:
|
|
raise RuntimeError('Cannot pause_reading() when closing')
|
|
if self._paused:
|
|
raise RuntimeError('Already paused')
|
|
self._paused = True
|
|
if self._loop.get_debug():
|
|
logger.debug("%r pauses reading", self)
|
|
|
|
def resume_reading(self):
|
|
if not self._paused:
|
|
raise RuntimeError('Not paused')
|
|
self._paused = False
|
|
if self._closing:
|
|
return
|
|
self._loop.call_soon(self._loop_reading, self._read_fut)
|
|
if self._loop.get_debug():
|
|
logger.debug("%r resumes reading", self)
|
|
|
|
def _loop_reading(self, fut=None):
|
|
if self._paused:
|
|
return
|
|
data = None
|
|
|
|
try:
|
|
if fut is not None:
|
|
assert self._read_fut is fut or (self._read_fut is None and
|
|
self._closing)
|
|
self._read_fut = None
|
|
data = fut.result() # deliver data later in "finally" clause
|
|
|
|
if self._closing:
|
|
# since close() has been called we ignore any read data
|
|
data = None
|
|
return
|
|
|
|
if data == b'':
|
|
# we got end-of-file so no need to reschedule a new read
|
|
return
|
|
|
|
# reschedule a new read
|
|
self._read_fut = self._loop._proactor.recv(self._sock, 4096)
|
|
except ConnectionAbortedError as exc:
|
|
if not self._closing:
|
|
self._fatal_error(exc, 'Fatal read error on pipe transport')
|
|
elif self._loop.get_debug():
|
|
logger.debug("Read error on pipe transport while closing",
|
|
exc_info=True)
|
|
except ConnectionResetError as exc:
|
|
self._force_close(exc)
|
|
except OSError as exc:
|
|
self._fatal_error(exc, 'Fatal read error on pipe transport')
|
|
except futures.CancelledError:
|
|
if not self._closing:
|
|
raise
|
|
else:
|
|
self._read_fut.add_done_callback(self._loop_reading)
|
|
finally:
|
|
if data:
|
|
self._protocol.data_received(data)
|
|
elif data is not None:
|
|
if self._loop.get_debug():
|
|
logger.debug("%r received EOF", self)
|
|
keep_open = self._protocol.eof_received()
|
|
if not keep_open:
|
|
self.close()
|
|
|
|
|
|
class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
|
|
transports.WriteTransport):
|
|
"""Transport for write pipes."""
|
|
|
|
def write(self, data):
|
|
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
raise TypeError('data argument must be byte-ish (%r)',
|
|
type(data))
|
|
if self._eof_written:
|
|
raise RuntimeError('write_eof() already called')
|
|
|
|
if not data:
|
|
return
|
|
|
|
if self._conn_lost:
|
|
if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
|
|
logger.warning('socket.send() raised exception.')
|
|
self._conn_lost += 1
|
|
return
|
|
|
|
# Observable states:
|
|
# 1. IDLE: _write_fut and _buffer both None
|
|
# 2. WRITING: _write_fut set; _buffer None
|
|
# 3. BACKED UP: _write_fut set; _buffer a bytearray
|
|
# We always copy the data, so the caller can't modify it
|
|
# while we're still waiting for the I/O to happen.
|
|
if self._write_fut is None: # IDLE -> WRITING
|
|
assert self._buffer is None
|
|
# Pass a copy, except if it's already immutable.
|
|
self._loop_writing(data=bytes(data))
|
|
elif not self._buffer: # WRITING -> BACKED UP
|
|
# Make a mutable copy which we can extend.
|
|
self._buffer = bytearray(data)
|
|
self._maybe_pause_protocol()
|
|
else: # BACKED UP
|
|
# Append to buffer (also copies).
|
|
self._buffer.extend(data)
|
|
self._maybe_pause_protocol()
|
|
|
|
def _loop_writing(self, f=None, data=None):
|
|
try:
|
|
assert f is self._write_fut
|
|
self._write_fut = None
|
|
self._pending_write = 0
|
|
if f:
|
|
f.result()
|
|
if data is None:
|
|
data = self._buffer
|
|
self._buffer = None
|
|
if not data:
|
|
if self._closing:
|
|
self._loop.call_soon(self._call_connection_lost, None)
|
|
if self._eof_written:
|
|
self._sock.shutdown(socket.SHUT_WR)
|
|
# Now that we've reduced the buffer size, tell the
|
|
# protocol to resume writing if it was paused. Note that
|
|
# we do this last since the callback is called immediately
|
|
# and it may add more data to the buffer (even causing the
|
|
# protocol to be paused again).
|
|
self._maybe_resume_protocol()
|
|
else:
|
|
self._write_fut = self._loop._proactor.send(self._sock, data)
|
|
if not self._write_fut.done():
|
|
assert self._pending_write == 0
|
|
self._pending_write = len(data)
|
|
self._write_fut.add_done_callback(self._loop_writing)
|
|
self._maybe_pause_protocol()
|
|
else:
|
|
self._write_fut.add_done_callback(self._loop_writing)
|
|
except ConnectionResetError as exc:
|
|
self._force_close(exc)
|
|
except OSError as exc:
|
|
self._fatal_error(exc, 'Fatal write error on pipe transport')
|
|
|
|
def can_write_eof(self):
|
|
return True
|
|
|
|
def write_eof(self):
|
|
self.close()
|
|
|
|
def abort(self):
|
|
self._force_close(None)
|
|
|
|
|
|
class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
|
|
def __init__(self, *args, **kw):
|
|
super().__init__(*args, **kw)
|
|
self._read_fut = self._loop._proactor.recv(self._sock, 16)
|
|
self._read_fut.add_done_callback(self._pipe_closed)
|
|
|
|
def _pipe_closed(self, fut):
|
|
if fut.cancelled():
|
|
# the transport has been closed
|
|
return
|
|
assert fut.result() == b''
|
|
if self._closing:
|
|
assert self._read_fut is None
|
|
return
|
|
assert fut is self._read_fut, (fut, self._read_fut)
|
|
self._read_fut = None
|
|
if self._write_fut is not None:
|
|
self._force_close(BrokenPipeError())
|
|
else:
|
|
self.close()
|
|
|
|
|
|
class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
|
|
_ProactorBaseWritePipeTransport,
|
|
transports.Transport):
|
|
"""Transport for duplex pipes."""
|
|
|
|
def can_write_eof(self):
|
|
return False
|
|
|
|
def write_eof(self):
|
|
raise NotImplementedError
|
|
|
|
|
|
class _ProactorSocketTransport(_ProactorReadPipeTransport,
|
|
_ProactorBaseWritePipeTransport,
|
|
transports.Transport):
|
|
"""Transport for connected sockets."""
|
|
|
|
def _set_extra(self, sock):
|
|
self._extra['socket'] = sock
|
|
try:
|
|
self._extra['sockname'] = sock.getsockname()
|
|
except (socket.error, AttributeError):
|
|
if self._loop.get_debug():
|
|
logger.warning("getsockname() failed on %r",
|
|
sock, exc_info=True)
|
|
if 'peername' not in self._extra:
|
|
try:
|
|
self._extra['peername'] = sock.getpeername()
|
|
except (socket.error, AttributeError):
|
|
if self._loop.get_debug():
|
|
logger.warning("getpeername() failed on %r",
|
|
sock, exc_info=True)
|
|
|
|
def can_write_eof(self):
|
|
return True
|
|
|
|
def write_eof(self):
|
|
if self._closing or self._eof_written:
|
|
return
|
|
self._eof_written = True
|
|
if self._write_fut is None:
|
|
self._sock.shutdown(socket.SHUT_WR)
|
|
|
|
|
|
class BaseProactorEventLoop(base_events.BaseEventLoop):
|
|
|
|
def __init__(self, proactor):
|
|
super().__init__()
|
|
logger.debug('Using proactor: %s', proactor.__class__.__name__)
|
|
self._proactor = proactor
|
|
self._selector = proactor # convenient alias
|
|
self._self_reading_future = None
|
|
self._accept_futures = {} # socket file descriptor => Future
|
|
proactor.set_loop(self)
|
|
self._make_self_pipe()
|
|
|
|
def _make_socket_transport(self, sock, protocol, waiter=None,
|
|
extra=None, server=None):
|
|
return _ProactorSocketTransport(self, sock, protocol, waiter,
|
|
extra, server)
|
|
|
|
def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None,
|
|
*, server_side=False, server_hostname=None,
|
|
extra=None, server=None):
|
|
if not sslproto._is_sslproto_available():
|
|
raise NotImplementedError("Proactor event loop requires Python 3.5"
|
|
" or newer (ssl.MemoryBIO) to support "
|
|
"SSL")
|
|
|
|
ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter,
|
|
server_side, server_hostname)
|
|
_ProactorSocketTransport(self, rawsock, ssl_protocol,
|
|
extra=extra, server=server)
|
|
return ssl_protocol._app_transport
|
|
|
|
def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
|
|
extra=None):
|
|
return _ProactorDuplexPipeTransport(self,
|
|
sock, protocol, waiter, extra)
|
|
|
|
def _make_read_pipe_transport(self, sock, protocol, waiter=None,
|
|
extra=None):
|
|
return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra)
|
|
|
|
def _make_write_pipe_transport(self, sock, protocol, waiter=None,
|
|
extra=None):
|
|
# We want connection_lost() to be called when other end closes
|
|
return _ProactorWritePipeTransport(self,
|
|
sock, protocol, waiter, extra)
|
|
|
|
def close(self):
|
|
if self.is_running():
|
|
raise RuntimeError("Cannot close a running event loop")
|
|
if self.is_closed():
|
|
return
|
|
|
|
# Call these methods before closing the event loop (before calling
|
|
# BaseEventLoop.close), because they can schedule callbacks with
|
|
# call_soon(), which is forbidden when the event loop is closed.
|
|
self._stop_accept_futures()
|
|
self._close_self_pipe()
|
|
self._proactor.close()
|
|
self._proactor = None
|
|
self._selector = None
|
|
|
|
# Close the event loop
|
|
super().close()
|
|
|
|
def sock_recv(self, sock, n):
|
|
return self._proactor.recv(sock, n)
|
|
|
|
def sock_sendall(self, sock, data):
|
|
return self._proactor.send(sock, data)
|
|
|
|
def sock_connect(self, sock, address):
|
|
try:
|
|
if self._debug:
|
|
base_events._check_resolved_address(sock, address)
|
|
except ValueError as err:
|
|
fut = futures.Future(loop=self)
|
|
fut.set_exception(err)
|
|
return fut
|
|
else:
|
|
return self._proactor.connect(sock, address)
|
|
|
|
def sock_accept(self, sock):
|
|
return self._proactor.accept(sock)
|
|
|
|
def _socketpair(self):
|
|
raise NotImplementedError
|
|
|
|
def _close_self_pipe(self):
|
|
if self._self_reading_future is not None:
|
|
self._self_reading_future.cancel()
|
|
self._self_reading_future = None
|
|
self._ssock.close()
|
|
self._ssock = None
|
|
self._csock.close()
|
|
self._csock = None
|
|
self._internal_fds -= 1
|
|
|
|
def _make_self_pipe(self):
|
|
# A self-socket, really. :-)
|
|
self._ssock, self._csock = self._socketpair()
|
|
self._ssock.setblocking(False)
|
|
self._csock.setblocking(False)
|
|
self._internal_fds += 1
|
|
self.call_soon(self._loop_self_reading)
|
|
|
|
def _loop_self_reading(self, f=None):
|
|
try:
|
|
if f is not None:
|
|
f.result() # may raise
|
|
f = self._proactor.recv(self._ssock, 4096)
|
|
except futures.CancelledError:
|
|
# _close_self_pipe() has been called, stop waiting for data
|
|
return
|
|
except Exception as exc:
|
|
self.call_exception_handler({
|
|
'message': 'Error on reading from the event loop self pipe',
|
|
'exception': exc,
|
|
'loop': self,
|
|
})
|
|
else:
|
|
self._self_reading_future = f
|
|
f.add_done_callback(self._loop_self_reading)
|
|
|
|
def _write_to_self(self):
|
|
self._csock.send(b'\0')
|
|
|
|
def _start_serving(self, protocol_factory, sock,
|
|
sslcontext=None, server=None):
|
|
|
|
def loop(f=None):
|
|
try:
|
|
if f is not None:
|
|
conn, addr = f.result()
|
|
if self._debug:
|
|
logger.debug("%r got a new connection from %r: %r",
|
|
server, addr, conn)
|
|
protocol = protocol_factory()
|
|
if sslcontext is not None:
|
|
self._make_ssl_transport(
|
|
conn, protocol, sslcontext, server_side=True,
|
|
extra={'peername': addr}, server=server)
|
|
else:
|
|
self._make_socket_transport(
|
|
conn, protocol,
|
|
extra={'peername': addr}, server=server)
|
|
if self.is_closed():
|
|
return
|
|
f = self._proactor.accept(sock)
|
|
except OSError as exc:
|
|
if sock.fileno() != -1:
|
|
self.call_exception_handler({
|
|
'message': 'Accept failed on a socket',
|
|
'exception': exc,
|
|
'socket': sock,
|
|
})
|
|
sock.close()
|
|
elif self._debug:
|
|
logger.debug("Accept failed on socket %r",
|
|
sock, exc_info=True)
|
|
except futures.CancelledError:
|
|
sock.close()
|
|
else:
|
|
self._accept_futures[sock.fileno()] = f
|
|
f.add_done_callback(loop)
|
|
|
|
self.call_soon(loop)
|
|
|
|
def _process_events(self, event_list):
|
|
# Events are processed in the IocpProactor._poll() method
|
|
pass
|
|
|
|
def _stop_accept_futures(self):
|
|
for future in self._accept_futures.values():
|
|
future.cancel()
|
|
self._accept_futures.clear()
|
|
|
|
def _stop_serving(self, sock):
|
|
self._stop_accept_futures()
|
|
self._proactor._stop_serving(sock)
|
|
sock.close()
|