"""
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from __future__ import annotations
import warnings
from typing import (
TYPE_CHECKING,
Any,
AsyncIterable,
Dict,
Iterable,
Optional,
Tuple,
Type,
Union,
cast,
)
from urllib.parse import urlparse
from w3lib.url import safe_url_string
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.settings import BaseSettings
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_unicode
from scrapy.utils.url import strip_url
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
LOCAL_SCHEMES: Tuple[str, ...] = (
"about",
"blob",
"data",
"filesystem",
)
POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
POLICY_STRICT_ORIGIN = "strict-origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy:
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES
name: str
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
raise NotImplementedError()
def stripped_referrer(self, url: str) -> Optional[str]:
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.strip_url(url)
return None
def origin_referrer(self, url: str) -> Optional[str]:
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.origin(url)
return None
def strip_url(self, url: str, origin_only: bool = False) -> Optional[str]:
"""
https://www.w3.org/TR/referrer-policy/#strip-url
If url is null, return no referrer.
If url's scheme is a local scheme, then return no referrer.
Set url's username to the empty string.
Set url's password to null.
Set url's fragment to null.
If the origin-only flag is true, then:
Set url's path to null.
Set url's query to null.
Return url.
"""
if not url:
return None
return strip_url(
url,
strip_credentials=True,
strip_fragment=True,
strip_default_port=True,
origin_only=origin_only,
)
def origin(self, url: str) -> Optional[str]:
"""Return serialized origin (scheme, host, path) for a request or response URL."""
return self.strip_url(url, origin_only=True)
def potentially_trustworthy(self, url: str) -> bool:
# Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
parsed_url = urlparse(url)
if parsed_url.scheme in ("data",):
return False
return self.tls_protected(url)
def tls_protected(self, url: str) -> bool:
return urlparse(url).scheme in ("https", "ftps")
[docs]class NoReferrerPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
The simplest policy is "no-referrer", which specifies that no referrer information
is to be sent along with requests made from a particular request client to any origin.
The header will be omitted entirely.
"""
name: str = POLICY_NO_REFERRER
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return None
[docs]class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
The "no-referrer-when-downgrade" policy sends a full URL along with requests
from a TLS-protected environment settings object to a potentially trustworthy URL,
and requests from clients which are not TLS-protected to any origin.
Requests from TLS-protected clients to non-potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
This is a user agent's default behavior, if no policy is otherwise specified.
"""
name: str = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if not self.tls_protected(response_url) or self.tls_protected(request_url):
return self.stripped_referrer(response_url)
return None
[docs]class SameOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
is sent as referrer information when making same-origin requests from a particular request client.
Cross-origin requests, on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name: str = POLICY_SAME_ORIGIN
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if self.origin(response_url) == self.origin(request_url):
return self.stripped_referrer(response_url)
return None
[docs]class OriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
The "origin" policy specifies that only the ASCII serialization
of the origin of the request client is sent as referrer information
when making both same-origin requests and cross-origin requests
from a particular request client.
"""
name: str = POLICY_ORIGIN
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return self.origin_referrer(response_url)
[docs]class StrictOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
The "strict-origin" policy sends the ASCII serialization
of the origin of the request client when making requests:
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
- from non-TLS-protected environment settings objects to any origin.
Requests from TLS-protected request clients to non- potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name: str = POLICY_STRICT_ORIGIN
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
if (
self.tls_protected(response_url)
and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
return None
[docs]class OriginWhenCrossOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
The "origin-when-cross-origin" policy specifies that a full URL,
stripped for use as a referrer, is sent as referrer information
when making same-origin requests from a particular request client,
and only the ASCII serialization of the origin of the request client
is sent as referrer information when making cross-origin requests
from a particular request client.
"""
name: str = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
return origin
[docs]class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
The "strict-origin-when-cross-origin" policy specifies that a full URL,
stripped for use as a referrer, is sent as referrer information
when making same-origin requests from a particular request client,
and only the ASCII serialization of the origin of the request client
when making cross-origin requests:
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
- from non-TLS-protected environment settings objects to any origin.
Requests from TLS-protected clients to non- potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name: str = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
if (
self.tls_protected(response_url)
and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
return None
[docs]class UnsafeUrlPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
is sent along with both cross-origin requests
and same-origin requests made from a particular request client.
Note: The policy's name doesn't lie; it is unsafe.
This policy will leak origins and paths from TLS-protected resources
to insecure origins.
Carefully consider the impact of setting such a policy for potentially sensitive documents.
"""
name: str = POLICY_UNSAFE_URL
def referrer(self, response_url: str, request_url: str) -> Optional[str]:
return self.stripped_referrer(response_url)
[docs]class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
"""
A variant of "no-referrer-when-downgrade",
with the addition that "Referer" is not sent if the parent request was
using ``file://`` or ``s3://`` scheme.
"""
NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES + ("file", "s3")
name: str = POLICY_SCRAPY_DEFAULT
_policy_classes: Dict[str, Type[ReferrerPolicy]] = {
p.name: p
for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)
}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[""] = NoReferrerWhenDowngradePolicy
def _load_policy_class(
policy: str, warning_only: bool = False
) -> Optional[Type[ReferrerPolicy]]:
"""
Expect a string for the path to the policy class,
otherwise try to interpret the string as a standard value
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return cast(Type[ReferrerPolicy], load_object(policy))
except ValueError:
try:
return _policy_classes[policy.lower()]
except KeyError:
msg = f"Could not load referrer policy {policy!r}"
if not warning_only:
raise RuntimeError(msg)
else:
warnings.warn(msg, RuntimeWarning)
return None
[docs]class RefererMiddleware:
def __init__(self, settings: Optional[BaseSettings] = None):
self.default_policy: Type[ReferrerPolicy] = DefaultReferrerPolicy
if settings is not None:
settings_policy = _load_policy_class(settings.get("REFERRER_POLICY"))
assert settings_policy
self.default_policy = settings_policy
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
if not crawler.settings.getbool("REFERER_ENABLED"):
raise NotConfigured
mw = cls(crawler.settings)
# Note: this hook is a bit of a hack to intercept redirections
crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
return mw
def policy(
self, resp_or_url: Union[Response, str], request: Request
) -> ReferrerPolicy:
"""
Determine Referrer-Policy to use from a parent Response (or URL),
and a Request to be sent.
- if a valid policy is set in Request meta, it is used.
- if the policy is set in meta but is wrong (e.g. a typo error),
the policy from settings is used
- if the policy is not set in Request meta,
but there is a Referrer-policy header in the parent response,
it is used if valid
- otherwise, the policy from settings is used.
"""
policy_name = request.meta.get("referrer_policy")
if policy_name is None:
if isinstance(resp_or_url, Response):
policy_header = resp_or_url.headers.get("Referrer-Policy")
if policy_header is not None:
policy_name = to_unicode(policy_header.decode("latin1"))
if policy_name is None:
return self.default_policy()
cls = _load_policy_class(policy_name, warning_only=True)
return cls() if cls else self.default_policy()
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
) -> Iterable[Any]:
return (self._set_referer(r, response) for r in result)
async def process_spider_output_async(
self, response: Response, result: AsyncIterable[Any], spider: Spider
) -> AsyncIterable[Any]:
async for r in result:
yield self._set_referer(r, response)
def _set_referer(self, r: Any, response: Response) -> Any:
if isinstance(r, Request):
referrer = self.policy(response, r).referrer(response.url, r.url)
if referrer is not None:
r.headers.setdefault("Referer", referrer)
return r
def request_scheduled(self, request: Request, spider: Spider) -> None:
# check redirected request to patch "Referer" header if necessary
redirected_urls = request.meta.get("redirect_urls", [])
if redirected_urls:
request_referrer = request.headers.get("Referer")
# we don't patch the referrer value if there is none
if request_referrer is not None:
# the request's referrer header value acts as a surrogate
# for the parent response URL
#
# Note: if the 3xx response contained a Referrer-Policy header,
# the information is not available using this hook
parent_url = safe_url_string(request_referrer)
policy_referrer = self.policy(parent_url, request).referrer(
parent_url, request.url
)
if policy_referrer != request_referrer.decode("latin1"):
if policy_referrer is None:
request.headers.pop("Referer")
else:
request.headers["Referer"] = policy_referrer