"""Download handlers for http and https schemes"""
from __future__ import annotations
import ipaddress
import logging
import re
from contextlib import suppress
from functools import partial
from io import BytesIO
from pathlib import Path
from time import monotonic
from typing import TYPE_CHECKING, Any, BinaryIO, NotRequired, TypedDict, TypeVar, cast
from urllib.parse import urldefrag
from twisted.internet import ssl
from twisted.internet.defer import Deferred, succeed
from twisted.internet.endpoints import (
HostnameEndpoint,
TCP6ClientEndpoint,
wrapClientTLS,
)
from twisted.internet.interfaces import IStreamClientEndpoint
from twisted.internet.protocol import ClientFactory, Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web._newclient import (
HEADER,
STATUS,
HTTP11ClientProtocol,
HTTPClientParser,
)
from twisted.web.client import (
URI,
Agent,
HTTPConnectionPool,
ResponseDone,
ResponseFailed,
_HTTP11ClientFactory,
)
from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
from zope.interface import implementer
from scrapy import Request, signals
from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacks
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.exceptions import (
DownloadCancelledError,
DownloadTimeoutError,
NotConfigured,
ResponseDataLossError,
StopDownload,
)
from scrapy.http import Headers, Response
from scrapy.utils._download_handlers import (
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
get_proxy_headers,
get_warnsize,
get_warnsize_msg,
make_response,
normalize_bind_address,
wrap_twisted_exceptions,
)
from scrapy.utils._ssl import _log_ssl_conn_debug_info
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
import os
from collections.abc import Callable
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IAddress, IConsumer, IProtocol
from twisted.web._newclient import Request as TxRequest
from scrapy.core.downloader.contextfactory import _PolicyForHTTPS
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class _ResultT(TypedDict):
txresponse: TxResponse
body: NotRequired[bytes]
flags: NotRequired[list[str] | None]
certificate: NotRequired[ssl.Certificate | None]
ip_address: NotRequired[ipaddress.IPv4Address | ipaddress.IPv6Address | None]
stop_download: NotRequired[StopDownload | None]
class _LenientHTTPClientParser(HTTPClientParser):
"""Response parser that skips bad response header lines, those with no
colon in them, instead of failing to parse the whole response.
Some servers send such lines, and web browsers skip them and keep parsing
the header lines that follow. See
https://github.com/scrapy/scrapy/issues/210.
"""
def lineReceived(self, line: bytes) -> None:
# A copy of twisted.web._newclient.HTTPParser.lineReceived() where the
# header name and value are only extracted from header lines that have
# a colon.
# Handle the normal CR LF case.
line = line.removesuffix(b"\r")
if self.state == STATUS:
self.statusReceived(line) # type: ignore[no-untyped-call]
self.state = HEADER
return
# HEADER is the only other state in which lines are received, as the
# parser switches to raw mode for the response body.
if not line or line[0] not in b" \t":
if self._partialHeader is not None:
header = b"".join(self._partialHeader)
if b":" in header:
name, value = header.split(b":", 1)
self.headerReceived(name, value.strip()) # type: ignore[no-untyped-call]
else:
logger.debug(
f"Skipping the bad response header line {header!r}, as "
f"it has no colon."
)
if not line:
# Empty line means the header section is over.
self.allHeadersReceived() # type: ignore[no-untyped-call]
else:
# Line not beginning with LWS is another header.
self._partialHeader = [line]
else:
# A line beginning with LWS is a continuation of a header begun on
# a previous line.
self._partialHeader.append(line) # type: ignore[union-attr]
class _LenientHTTP11ClientProtocol(HTTP11ClientProtocol):
"""Protocol that parses responses with :class:`_LenientHTTPClientParser`."""
def request(self, request: TxRequest) -> Deferred[IResponse]:
d: Deferred[IResponse] = super().request(request)
# HTTP11ClientProtocol.request() hardcodes the parser class, so the
# only way to use a different one is to replace the class of the parser
# object that it creates. This is safe because
# _LenientHTTPClientParser defines no additional state. The parser is
# always there because HTTPConnectionPool only reuses connections whose
# protocol is in the QUIESCENT state, for which request() always
# creates a parser.
assert self._parser is not None
self._parser.__class__ = _LenientHTTPClientParser
# For responses without a body, twisted.web.client.Response never
# hands its transport to a protocol, so the certificate and IP
# address cannot be read from it later (see
# _ResponseReader.connectionMade). self.transport, however, is the
# connection's real transport and outlives any single request, so
# read the certificate and IP address from it directly and stash
# them on the response.
assert self.transport is not None
transport = self.transport
def _attach_connection_info(response: IResponse) -> IResponse:
with suppress(AttributeError):
response._scrapy_certificate = ssl.Certificate(
transport.getPeerCertificate()
)
with suppress(AttributeError):
response._scrapy_ip_address = ipaddress.ip_address(
transport.getPeer().host
)
return response
d.addCallback(_attach_connection_info)
return d
class _TrackedHTTP11ClientProtocol(_LenientHTTP11ClientProtocol):
"""Connection that tells the pool that opened it when it closes, whether it
was cached at the time or in use."""
_pool: _BoundedHTTPConnectionPool
_pool_key: Any
def connectionLost(self, reason: Failure = connectionDone) -> None:
self._pool._forget(self)
super().connectionLost(reason) # type: ignore[no-untyped-call]
class _BoundedHTTP11ClientFactory(_HTTP11ClientFactory):
noisy = False
def __init__(
self,
quiescentCallback: Callable[[HTTP11ClientProtocol], None],
metadata: str,
pool: _BoundedHTTPConnectionPool,
key: Any,
):
super().__init__(quiescentCallback, metadata) # type: ignore[no-untyped-call]
self._pool: _BoundedHTTPConnectionPool = pool
self._key: Any = key
def buildProtocol(self, addr: IAddress | None) -> _TrackedHTTP11ClientProtocol:
protocol = _TrackedHTTP11ClientProtocol(self._quiescentCallback) # type: ignore[no-untyped-call]
protocol._pool = self._pool
protocol._pool_key = self._key
self._pool._open.add(protocol)
return protocol
class _BoundedHTTPConnectionPool(HTTPConnectionPool):
"""Connection pool that keeps the number of open connections, cached and in
use alike, under *limit*, where 0 means no limit.
Opening a connection once the limit is reached closes the least recently
used cached connection to make room for it. When there is no cached
connection left to close, the limit is exceeded rather than requests
delayed.
"""
_factory = _BoundedHTTP11ClientFactory # type: ignore[assignment]
def __init__(self, reactor: ReactorBase, limit: int, keepalive_timeout: float):
super().__init__(reactor, persistent=bool(keepalive_timeout)) # type: ignore[no-untyped-call]
self.cachedConnectionTimeout = keepalive_timeout # type: ignore[assignment]
self._limit: int = limit
self._open: set[_TrackedHTTP11ClientProtocol] = set()
def _newConnection(
self, key: Any, endpoint: IStreamClientEndpoint
) -> Deferred[HTTP11ClientProtocol]:
while self._limit and len(self._open) >= self._limit and self._evict():
pass
def quiescentCallback(protocol: HTTP11ClientProtocol) -> None:
self._putConnection(key, protocol) # type: ignore[no-untyped-call]
factory = self._factory(quiescentCallback, repr(endpoint), self, key)
return endpoint.connect(factory) # type: ignore[no-any-return]
def _forget(self, connection: _TrackedHTTP11ClientProtocol) -> None:
self._open.discard(connection)
def _evict(self) -> bool:
"""Close the least recently used cached connection, and report whether
there was one to close.
``_timeouts`` holds cached connections only, and ``getConnection()``
drops the ones it hands out, so its insertion order goes from least to
most recently used.
"""
connection = next(iter(self._timeouts), None)
if connection is None:
return False
self._timeouts[connection].cancel()
# dropped early so that a caller looping on this method sees the room
# made by connections that have yet to finish closing
self._forget(connection)
key = connection._pool_key
self._removeConnection(key, connection) # type: ignore[no-untyped-call]
if not self._connections[key]:
del self._connections[key]
return True
[docs]
class HTTP11DownloadHandler(BaseHttpDownloadHandler):
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):
raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
_install_twisted_idna_fallbacks()
super().__init__(crawler)
self._crawler = crawler
from twisted.internet import reactor
self._pool: _BoundedHTTPConnectionPool = _BoundedHTTPConnectionPool(
reactor, self._pool_size_total, self._keepalive_timeout
)
self._pool.maxPersistentPerHost = self._pool_size_per_host
self._contextFactory: _PolicyForHTTPS = _load_context_factory_from_settings(
crawler
)
self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
self._disconnect_timeout: int = 1
async def download_request(self, request: Request) -> Response:
if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover
warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover
warn_on_deprecated_spider_attribute(
"download_warnsize", "DOWNLOAD_WARNSIZE"
)
agent = _ScrapyAgent(
contextFactory=self._contextFactory,
bindAddress=self._bind_address,
pool=self._pool,
maxsize=getattr(
self._crawler.spider, "download_maxsize", self._default_maxsize
),
warnsize=getattr(
self._crawler.spider, "download_warnsize", self._default_warnsize
),
fail_on_dataloss=self._fail_on_dataloss,
crawler=self._crawler,
tls_verbose_logging=self._tls_verbose_logging,
)
try:
with wrap_twisted_exceptions():
return await maybe_deferred_to_future(agent.download_request(request))
except ResponseDataLossError:
if not self._fail_on_dataloss_warned:
logger.warning(get_dataloss_msg(request.url))
self._fail_on_dataloss_warned = True
raise
async def close(self) -> None:
from twisted.internet import reactor
d: Deferred[None] = self._pool.closeCachedConnections() # type: ignore[no-untyped-call]
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
#
# Twisted issue addressing this problem can be found here:
# https://github.com/twisted/twisted/issues/7738
#
# closeCachedConnections doesn't handle external errbacks, so we'll
# issue a callback after `_disconnect_timeout` seconds.
#
# See also https://github.com/scrapy/scrapy/issues/2653
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ())
try:
await maybe_deferred_to_future(d)
finally:
# Only inactive if the timeout above won the race, which the tests
# cannot force.
if delayed_call.active(): # pragma: no branch
delayed_call.cancel()
class TunnelError(Exception):
"""An HTTP CONNECT tunnel could not be established by the proxy."""
def _is_ipv6_literal(host: str) -> bool:
try:
return isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address)
except ValueError: # a hostname
return False
class _TunnelProtocol(Protocol):
"""Sends an HTTP CONNECT request to a proxy and, once the tunnel is open,
hands the connection over to a wrapped protocol.
It plays the role of Twisted's ``_WrappingProtocol`` but performs a CONNECT
handshake before connecting the wrapped protocol. The wrapped protocol is
the destination TLS protocol, so its handshake is negotiated inside the
tunnel; when the proxy itself is reached over TLS this results in a
TLS-in-TLS connection.
"""
_truncatedLength = 1000
_responseAnswer = (
r"HTTP/1\.. (?P<status>\d{3})(?P<reason>.{," + str(_truncatedLength) + r"})"
)
_responseMatcher = re.compile(_responseAnswer.encode())
def __init__(
self,
connectedDeferred: Deferred[IProtocol],
wrappedProtocol: IProtocol,
host: bytes,
port: int,
proxyHost: str,
proxyPort: int,
proxyHeaders: tuple[tuple[str, str], ...] = (),
):
self._connectedDeferred = connectedDeferred
self._wrappedProtocol = wrappedProtocol
self._host = host
self._port = port
self._proxyHost = proxyHost
self._proxyPort = proxyPort
self._proxyHeaders = proxyHeaders
self._connectBuffer: bytearray = bytearray()
self._tunnelReady = False
@staticmethod
def _format_host(host: bytes) -> bytes:
"""Return *host* as it goes into the authority of the CONNECT request,
i.e. with square brackets around IPv6 literals.
"""
# URI.host strips the brackets of IPv6 literals, and only those contain
# colons.
return b"[" + host + b"]" if b":" in host else host
def logPrefix(self) -> str:
return type(self._wrappedProtocol).__name__
def connectionMade(self) -> None:
"""Asks the proxy to open a tunnel."""
assert self.transport
self.transport.write(
_tunnel_request_data(
self._format_host(self._host), self._port, self._proxyHeaders
)
)
def dataReceived(self, data: bytes) -> None:
if self._tunnelReady:
self._wrappedProtocol.dataReceived(data)
return
if self._connectedDeferred.called:
# The tunnel was cancelled while waiting for the CONNECT response.
return
self._connectBuffer += data
# make sure that enough (all) bytes are consumed
# and that we've got all HTTP headers (ending with a blank line)
# from the proxy so that we don't send those bytes to the TLS layer
#
# see https://github.com/scrapy/scrapy/issues/2491
if b"\r\n\r\n" not in self._connectBuffer:
return
header, _, remaining = bytes(self._connectBuffer).partition(b"\r\n\r\n")
respm = self._responseMatcher.match(header)
if respm and int(respm.group("status")) == 200:
self._tunnelReady = True
# Hand the connection over to the wrapped (destination TLS)
# protocol, which starts its handshake through the tunnel.
assert self.transport is not None
self._wrappedProtocol.makeConnection(self.transport)
self._connectedDeferred.callback(self._wrappedProtocol)
if remaining:
self._wrappedProtocol.dataReceived(remaining)
else:
extra: Any
if respm:
extra = {
"status": int(respm.group("status")),
"reason": respm.group("reason").strip(),
}
else:
extra = header[: self._truncatedLength]
self._connectedDeferred.errback(
TunnelError(
"Could not open CONNECT tunnel with proxy "
f"{self._proxyHost}:{self._proxyPort} [{extra!r}]"
)
)
def connectionLost(self, reason: Failure = connectionDone) -> None:
if self._tunnelReady:
self._wrappedProtocol.connectionLost(reason)
elif not self._connectedDeferred.called:
# The connection was lost before the tunnel was established (e.g.
# the proxy closed the connection in response to CONNECT).
self._connectedDeferred.errback(reason)
class _TunnelFactory(ClientFactory):
"""Builds :class:`_TunnelProtocol` instances wrapping the protocols built
by another factory."""
def __init__(
self,
connectedDeferred: Deferred[IProtocol],
wrappedFactory: Factory,
host: bytes,
port: int,
proxyHost: str,
proxyPort: int,
proxyHeaders: tuple[tuple[str, str], ...] = (),
):
self._connectedDeferred = connectedDeferred
self._wrappedFactory = wrappedFactory
self._host = host
self._port = port
self._proxyHost = proxyHost
self._proxyPort = proxyPort
self._proxyHeaders = proxyHeaders
def doStart(self) -> None:
self._wrappedFactory.doStart()
def doStop(self) -> None:
self._wrappedFactory.doStop()
def buildProtocol(self, addr: IAddress) -> _TunnelProtocol:
wrappedProtocol = self._wrappedFactory.buildProtocol(addr)
# The wrapped factory is always the destination TLS factory built by
# wrapClientTLS, whose buildProtocol never returns None.
assert wrappedProtocol is not None
return _TunnelProtocol(
self._connectedDeferred,
wrappedProtocol,
self._host,
self._port,
self._proxyHost,
self._proxyPort,
self._proxyHeaders,
)
@implementer(IStreamClientEndpoint)
class _TunnelEndpoint:
"""A wrapper endpoint that opens an HTTP CONNECT tunnel over the connection
established by the wrapped endpoint before delivering it to the protocol
factory.
The wrapped endpoint connects to the proxy (over plain TCP for ``http://``
proxies, or over TLS via :func:`~twisted.internet.endpoints.wrapClientTLS`
for ``https://`` proxies). Wrapping this endpoint in turn with
``wrapClientTLS`` negotiates the destination TLS session inside the tunnel.
"""
def __init__(
self,
wrappedEndpoint: IStreamClientEndpoint,
host: bytes,
port: int,
proxyHost: str,
proxyPort: int,
proxyHeaders: tuple[tuple[str, str], ...] = (),
):
self._wrappedEndpoint = wrappedEndpoint
self._host = host
self._port = port
self._proxyHost = proxyHost
self._proxyPort = proxyPort
self._proxyHeaders = proxyHeaders
def connect(self, protocolFactory: Factory) -> Deferred[IProtocol]:
proxyProtocol: IProtocol | None = None
def proxyConnected(protocol: IProtocol) -> None:
nonlocal proxyProtocol
proxyProtocol = protocol
def cancel(_: Deferred[IProtocol]) -> None:
# Tear down the connection to the proxy, e.g. on a download timeout.
if proxyProtocol is None:
# Still connecting to the proxy. This errbacks
# connectedDeferred through _connectFailed().
proxyDeferred.cancel()
else:
assert isinstance(proxyProtocol, _TunnelProtocol)
assert proxyProtocol.transport
proxyProtocol.transport.loseConnection()
connectedDeferred: Deferred[IProtocol] = Deferred(cancel)
tunnelFactory = _TunnelFactory(
connectedDeferred,
protocolFactory,
self._host,
self._port,
self._proxyHost,
self._proxyPort,
self._proxyHeaders,
)
proxyDeferred = self._wrappedEndpoint.connect(tunnelFactory)
proxyDeferred.addCallbacks(
proxyConnected, self._connectFailed, errbackArgs=(connectedDeferred,)
)
return connectedDeferred
@staticmethod
def _connectFailed(
failure: Failure, connectedDeferred: Deferred[IProtocol]
) -> None:
# Reached only when connecting to the proxy fails, i.e. before the
# tunnel deferred has been fired.
connectedDeferred.errback(failure)
def _tunnel_request_data(
host: bytes, port: int, proxy_headers: tuple[tuple[str, str], ...] = ()
) -> bytes:
r"""
Return binary content of a CONNECT request.
>>> _tunnel_request_data(b"example.com", 8080).decode()
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> _tunnel_request_data(b"example.com", 8080, (("Proxy-Authorization", "123"),)).decode()
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
"""
host_value = host + b":" + str(port).encode()
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
tunnel_req += b"Host: " + host_value + b"\r\n"
for name, value in proxy_headers:
tunnel_req += name.encode() + b": " + value.encode() + b"\r\n"
tunnel_req += b"\r\n"
return tunnel_req
class _BindAddressAgent(Agent):
"""An Agent that adds the configured local bind address to the
connection pool key, so pooled connections are not shared between
requests bound to different addresses."""
def __init__(
self,
reactor: ReactorBase,
contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call]
def _requestWithEndpoint(
self,
key: tuple[Any, ...],
endpoint: IStreamClientEndpoint,
method: bytes,
parsedURI: URI,
headers: TxHeaders | None,
bodyProducer: IBodyProducer | None,
requestPath: bytes,
) -> Deferred[IResponse]:
key += (self._endpointFactory._bindAddress,)
return super()._requestWithEndpoint(
key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
)
class _TunnelingAgent(_BindAddressAgent):
"""An agent that tunnels HTTPS downloads through a proxy using HTTP
CONNECT. It may look strange that we have chosen to subclass Agent and not
ProxyAgent but consider that after the tunnel is opened the proxy is
transparent to the client; thus the agent should behave like there is no
proxy involved.
``proxyConf`` is a ``(host, port, headers, tls)`` tuple, where ``tls``
indicates whether the connection to the proxy itself must be made over TLS
(an ``https://`` proxy). When it is, the destination TLS session runs inside
the proxy TLS session (TLS-in-TLS).
"""
def __init__(
self,
*,
reactor: ReactorBase,
proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool],
contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf: tuple[str, int, tuple[tuple[str, str], ...], bool] = proxyConf
self._contextFactory: _PolicyForHTTPS = contextFactory
def _getEndpoint(self, uri: URI) -> IStreamClientEndpoint:
proxyHost, proxyPort, proxyHeaders, proxyTLS = self._proxyConf
timeout = self._endpointFactory._connectTimeout
bindAddress = self._endpointFactory._bindAddress
# Reach a proxy given as a name through a HostnameEndpoint, the same
# endpoint Twisted's Agent uses: it resolves the name with
# reactor.nameResolver and picks the address family from the result, so
# proxies that only have AAAA records work, given an IPv6-capable
# TWISTED_DNS_RESOLVER. A literal IPv6 address skips name resolution
# instead, as the default resolver would fail to resolve it.
endpoint: IStreamClientEndpoint = (
TCP6ClientEndpoint(
self._reactor, proxyHost, proxyPort, timeout, bindAddress
)
if _is_ipv6_literal(proxyHost)
else HostnameEndpoint(
self._reactor,
proxyHost,
proxyPort,
timeout=timeout,
bindAddress=bindAddress,
)
)
if proxyTLS:
# Set up TLS with the proxy itself, so that the CONNECT request and
# the tunneled traffic are sent over it (TLS-in-TLS).
proxyCreator = self._contextFactory.creatorForNetloc(
proxyHost.encode("ascii"), proxyPort
)
endpoint = wrapClientTLS(proxyCreator, endpoint)
endpoint = _TunnelEndpoint(
endpoint, uri.host, uri.port, proxyHost, proxyPort, proxyHeaders
)
# Set proper Server Name Indication extension for the destination.
destCreator = self._contextFactory.creatorForNetloc(uri.host, uri.port)
return wrapClientTLS(destCreator, endpoint)
def _requestWithEndpoint(
self,
key: Any,
endpoint: IStreamClientEndpoint,
method: bytes,
parsedURI: URI,
headers: TxHeaders | None,
bodyProducer: IBodyProducer | None,
requestPath: bytes,
) -> Deferred[IResponse]:
# the proxy host and port and the headers sent to the proxy are part of
# the HTTP pool `key`, otherwise a request for the same remote host
# could reuse a tunnel that was opened through a different proxy or
# with different proxy headers
key += self._proxyConf
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
parsedURI=parsedURI,
headers=headers,
bodyProducer=bodyProducer,
requestPath=requestPath,
)
class _ScrapyProxyAgent(_BindAddressAgent):
def __init__(
self,
reactor: ReactorBase,
proxyURI: bytes,
contextFactory: _PolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__(
reactor=reactor,
contextFactory=contextFactory,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
pool=pool,
)
self._proxyURI: URI = URI.fromBytes(proxyURI)
def request(
self,
method: bytes,
uri: bytes,
headers: TxHeaders | None = None,
bodyProducer: IBodyProducer | None = None,
) -> Deferred[IResponse]:
"""
Issue a new request via the configured proxy.
"""
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
return self._requestWithEndpoint(
key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),
endpoint=self._getEndpoint(self._proxyURI), # type: ignore[no-untyped-call]
method=method,
parsedURI=URI.fromBytes(uri),
headers=headers,
bodyProducer=bodyProducer,
requestPath=uri,
)
class _ScrapyAgent:
def __init__(
self,
*,
contextFactory: _PolicyForHTTPS,
connectTimeout: float = 10,
bindAddress: str | tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
maxsize: int = 0,
warnsize: int = 0,
fail_on_dataloss: bool = True,
crawler: Crawler,
tls_verbose_logging: bool = False,
):
self._contextFactory: _PolicyForHTTPS = contextFactory
self._connectTimeout: float = connectTimeout
self._bindAddress: str | tuple[str, int] | None = bindAddress
self._pool: HTTPConnectionPool | None = pool
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
self._txresponse: TxResponse | None = None
self._crawler: Crawler = crawler
self._tls_verbose_logging: bool = tls_verbose_logging
def _get_agent(self, request: Request, timeout: float) -> Agent:
from twisted.internet import reactor
bindaddress = request.meta.get("bindaddress") or self._bindAddress
bindaddress = normalize_bind_address(bindaddress)
proxy = request.meta.get("proxy")
if proxy:
proxy = _add_http_if_no_scheme(proxy)
proxy_parsed = urlparse_cached(proxy)
proxy_host = proxy_parsed.hostname
proxy_port = proxy_parsed.port
if not proxy_port:
proxy_port = 443 if proxy_parsed.scheme == "https" else 80
if urlparse_cached(request).scheme == "https":
assert proxy_host is not None
proxy_headers = get_proxy_headers(request)
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
if proxyAuth and not any(
name == "Proxy-Authorization" for name, _ in proxy_headers
):
proxy_headers = (
("Proxy-Authorization", proxyAuth.decode()),
*proxy_headers,
)
proxyConf = (
proxy_host,
proxy_port,
proxy_headers,
proxy_parsed.scheme == "https",
)
return _TunnelingAgent(
reactor=reactor,
proxyConf=proxyConf,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
return _ScrapyProxyAgent(
reactor=reactor,
proxyURI=proxy.encode("ascii"),
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
return _BindAddressAgent(
reactor=reactor,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool,
)
def download_request(self, request: Request) -> Deferred[Response]:
from twisted.internet import reactor
timeout = request.meta.get("download_timeout") or self._connectTimeout
agent = self._get_agent(request, timeout)
# request details
url = urldefrag(request.url)[0]
method = request.method.encode()
headers = TxHeaders(request.headers)
if isinstance(agent, _TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
elif isinstance(agent, _ScrapyProxyAgent):
# without a tunnel the proxy reads the request headers, so the
# headers meant for it travel among them, and it is up to the proxy
# not to pass them on to the target server
for name, value in get_proxy_headers(request):
headers.addRawHeader(name, value)
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
body_file = request.meta.get("body_file")
bodybuf: BinaryIO = Path(body_file).open("wb") if body_file else BytesIO()
start_time = monotonic()
d: Deferred[IResponse] = agent.request(
method,
url.encode("ascii"),
headers,
cast("IBodyProducer", bodyproducer),
)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request, bodybuf)
d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url, body_file)
# check download timeout
self._timeout_cl = reactor.callLater(timeout, self._cb_timed_out, d3, request)
d3.addBoth(self._cb_timeout, request, url, timeout)
d3.addBoth(self._cb_close, bodybuf)
return d3
@staticmethod
def _cb_close(result: _T, bodybuf: BinaryIO) -> _T:
bodybuf.close()
return result
def _cb_timed_out(self, d3: Deferred[Response], request: Request) -> None:
fail_on_dataloss = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if not fail_on_dataloss and self._txresponse is not None:
# let the response be delivered as a partial one instead of
# cancelling the request outright
self._txresponse._transport.stopProducing()
else:
d3.cancel()
def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
if self._timeout_cl.active():
self._timeout_cl.cancel()
return result
# needed for HTTPS requests, otherwise _ResponseReader doesn't
# receive connectionLost()
if self._txresponse:
self._txresponse._transport.stopProducing()
if not isinstance(result, Failure):
return result
raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
request.meta["download_latency"] = monotonic() - start_time
return result
@staticmethod
def _headers_from_twisted_response(response: TxResponse) -> Headers:
headers = Headers()
if response.length != UNKNOWN_LENGTH:
headers[b"Content-Length"] = str(response.length).encode()
headers.update(response.headers.getAllRawHeaders())
return headers
def _cb_bodyready(
self, txresponse: TxResponse, request: Request, bodybuf: BinaryIO
) -> _ResultT | Deferred[_ResultT]:
if stop_download := check_stop_download(
signals.headers_received,
self._crawler,
request,
headers=self._headers_from_twisted_response(txresponse),
body_length=txresponse.length,
):
txresponse._transport.stopProducing()
txresponse._transport.loseConnection()
return {
"txresponse": txresponse,
"stop_download": stop_download,
}
# deliverBody hangs for responses without body
if cast("int", txresponse.length) == 0:
return {
"txresponse": txresponse,
"certificate": getattr(txresponse, "_scrapy_certificate", None),
"ip_address": getattr(txresponse, "_scrapy_ip_address", None),
}
maxsize = request.meta.get("download_maxsize", self._maxsize)
warnsize = get_warnsize(request.meta, self._warnsize)
expected_size = (
cast("int", txresponse.length)
if txresponse.length != UNKNOWN_LENGTH
else -1
)
fail_on_dataloss = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if maxsize and expected_size > maxsize:
warning_msg = get_maxsize_msg(
expected_size, maxsize, request, expected=True
)
logger.warning(warning_msg)
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
raise DownloadCancelledError(warning_msg)
reached_warnsize = bool(warnsize and expected_size > warnsize)
if reached_warnsize:
logger.warning(
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
txresponse.deliverBody(
_ResponseReader(
finished=d,
txresponse=txresponse,
request=request,
bodybuf=bodybuf,
maxsize=maxsize,
warnsize=warnsize,
reached_warnsize=reached_warnsize,
fail_on_dataloss=fail_on_dataloss,
crawler=self._crawler,
tls_verbose_logging=self._tls_verbose_logging,
)
)
# save response for timeouts
self._txresponse = txresponse
return d
@staticmethod
def _cancel(_: Any, txresponse: TxResponse) -> None:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
def _cb_bodydone(
self,
result: _ResultT,
url: str,
body_file: str | os.PathLike[str] | None,
) -> Response:
headers = self._headers_from_twisted_response(result["txresponse"])
version = result["txresponse"].version
protocol = f"{version[0].decode()}/{version[1]}.{version[2]}"
return make_response(
url=url,
status=int(result["txresponse"].code),
headers=headers,
body=result.get("body", b""),
flags=result.get("flags"),
certificate=result.get("certificate"),
ip_address=result.get("ip_address"),
protocol=protocol,
stop_download=result.get("stop_download"),
body_file=body_file,
)
@implementer(IBodyProducer)
class _RequestBodyProducer:
def __init__(self, body: bytes):
self.body = body
self.length = len(body)
def startProducing(self, consumer: IConsumer) -> Deferred[None]:
consumer.write(self.body)
return succeed(None)
def pauseProducing(self) -> None:
pass
def stopProducing(self) -> None:
pass
class _ResponseReader(Protocol):
def __init__(
self,
finished: Deferred[_ResultT],
txresponse: TxResponse,
request: Request,
bodybuf: BinaryIO,
maxsize: int,
warnsize: int,
fail_on_dataloss: bool,
crawler: Crawler,
*,
reached_warnsize: bool = False,
tls_verbose_logging: bool = False,
):
self._finished: Deferred[_ResultT] = finished
self._txresponse: TxResponse = txresponse
self._request: Request = request
self._bodybuf: BinaryIO = bodybuf
self._maxsize: int = maxsize
self._warnsize: int = warnsize
self._fail_on_dataloss: bool = fail_on_dataloss
self._reached_warnsize: bool = reached_warnsize
self._bytes_received: int = 0
self._certificate: ssl.Certificate | None = None
self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None
self._crawler: Crawler = crawler
self._tls_verbose_logging: bool = tls_verbose_logging
def _finish_response(
self, flags: list[str] | None = None, stop_download: StopDownload | None = None
) -> None:
self._finished.callback(
{
"txresponse": self._txresponse,
"body": (
self._bodybuf.getvalue()
if isinstance(self._bodybuf, BytesIO)
else b""
),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"stop_download": stop_download,
}
)
def connectionMade(self) -> None:
assert self.transport
with suppress(AttributeError):
self._certificate = ssl.Certificate(
self.transport._producer.getPeerCertificate()
)
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
if self._tls_verbose_logging:
connection = self.transport._producer.getHandle()
hostname = urlparse_cached(self._request).hostname
assert hostname is not None
_log_ssl_conn_debug_info(hostname, connection)
def dataReceived(self, data: bytes) -> None:
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
assert self.transport
self._bodybuf.write(data)
self._bytes_received += len(data)
if stop_download := check_stop_download(
signals.bytes_received, self._crawler, self._request, data=data
):
self.transport.stopProducing()
self.transport.loseConnection()
self._finish_response(stop_download=stop_download)
if self._maxsize and self._bytes_received > self._maxsize:
logger.warning(
get_maxsize_msg(
self._bytes_received, self._maxsize, self._request, expected=False
)
)
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
if (
self._warnsize
and self._bytes_received > self._warnsize
and not self._reached_warnsize
):
self._reached_warnsize = True
logger.warning(
get_warnsize_msg(
self._bytes_received, self._warnsize, self._request, expected=False
)
)
def connectionLost(self, reason: Failure = connectionDone) -> None:
if self._finished.called:
return
if reason.check(ResponseDone):
self._finish_response()
return
if reason.check(PotentialDataLoss):
self._finish_response(flags=["partial"])
return
# Twisted ends a response body in one of three ways: ResponseDone,
# PotentialDataLoss, or a ResponseFailed wrapping _DataLoss. See
# twisted.web._newclient.HTTPClientParser.connectionLost().
assert isinstance(reason.value, ResponseFailed)
assert any(r.check(_DataLoss) for r in reason.value.reasons)
if not self._fail_on_dataloss:
self._finish_response(flags=["dataloss"])
return
exc = ResponseDataLossError()
exc.__cause__ = reason.value
self._finished.errback(exc)