from __future__ import annotations
import logging
import pprint
import re
import sys
import warnings
from collections.abc import MutableMapping
from logging.config import dictConfig
from typing import TYPE_CHECKING, Any, cast
from twisted.internet import asyncioreactor
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils._colorize import tty_supports_color
from scrapy.utils._versions import get_versions
from scrapy.utils.misc import load_object
if TYPE_CHECKING:
from types import TracebackType
from scrapy.crawler import Crawler
from scrapy.logformatter import LogFormatterResult
logger = logging.getLogger(__name__)
def failure_to_exc_info(
failure: Failure,
) -> tuple[type[BaseException], BaseException, TracebackType | None] | None:
"""Extract exc_info from Failure instances"""
if isinstance(failure, Failure):
assert failure.type
assert failure.value
return (
failure.type,
failure.value,
cast("TracebackType | None", failure.getTracebackObject()),
)
return None
class TopLevelFormatter(logging.Filter):
"""Keep only top level loggers' name (direct children from root) from
records.
This filter will replace Scrapy loggers' names with 'scrapy'. This mimics
the old Scrapy log behaviour and helps shortening long names.
Since it can't be set for just one logger (it won't propagate for its
children), it's going to be set in the root handler, with a parametrized
``loggers`` list where it should act.
"""
def __init__(self, loggers: list[str] | None = None):
super().__init__()
self.loggers: list[str] = loggers or []
def filter(self, record: logging.LogRecord) -> bool:
if any(record.name.startswith(logger + ".") for logger in self.loggers):
record.name = record.name.split(".", 1)[0]
return True
class _SpiderRecordDefaultFilter(logging.Filter):
"""Default the ``spider`` log record attribute to ``"-"``.
This lets :setting:`LOG_FORMAT` reference ``%(spider)s`` even for log
records that were not given a ``spider`` via ``extra``.
"""
def filter(self, record: logging.LogRecord) -> bool:
if not hasattr(record, "spider"):
record.spider = "-"
return True
DEFAULT_LOGGING: dict[str, Any] = {
"version": 1,
"disable_existing_loggers": False,
"loggers": {
"filelock": {
"level": "ERROR",
},
"hpack": {
"level": "ERROR",
},
"httpcore": {
"level": "ERROR",
},
"httpx": {
"level": "WARNING",
},
"parso": {
"level": "ERROR",
},
"scrapy": {
"level": "DEBUG",
},
"twisted": {
"level": "ERROR",
},
},
}
def _configure_logger_levels(settings: Settings) -> None:
loggers = DEFAULT_LOGGING["loggers"] | {
name: {"level": level} for name, level in settings.getdict("LOG_LEVELS").items()
}
dictConfig(DEFAULT_LOGGING | {"loggers": loggers})
_scrapy_root_handler: logging.Handler | None = None
def install_scrapy_root_handler(settings: Settings) -> None:
global _scrapy_root_handler # noqa: PLW0603
_uninstall_scrapy_root_handler()
logging.root.setLevel(logging.NOTSET)
_scrapy_root_handler = _get_handler(settings)
logging.root.addHandler(_scrapy_root_handler)
def _uninstall_scrapy_root_handler() -> None:
global _scrapy_root_handler # noqa: PLW0603
if _scrapy_root_handler is None:
return
if _scrapy_root_handler in logging.root.handlers:
logging.root.removeHandler(_scrapy_root_handler)
_scrapy_root_handler.close()
_scrapy_root_handler = None
def get_scrapy_root_handler() -> logging.Handler | None:
return _scrapy_root_handler
class _ColoredFormatter(logging.Formatter):
"""Colorizes the output of *formatter* by log level with *colorizer*, a
:class:`colorlog.ColoredFormatter` that only renders the message.
"""
def __init__(self, formatter: logging.Formatter, colorizer: logging.Formatter):
super().__init__()
self._formatter = formatter
self._colorizer = colorizer
def format(self, record: logging.LogRecord) -> str:
formatted = logging.makeLogRecord(
{
**record.__dict__,
"msg": self._formatter.format(record),
"args": None,
"exc_info": None,
"exc_text": None,
"stack_info": None,
}
)
return self._colorizer.format(formatted)
def _get_formatter(handler: logging.Handler, settings: Settings) -> logging.Formatter:
formatter: logging.Formatter = load_object(settings["LOG_ROOT_FORMATTER"])(settings)
if (
isinstance(handler, logging.StreamHandler)
and not isinstance(handler, logging.FileHandler)
and settings.getbool("LOG_COLOR")
and handler.stream.isatty()
and tty_supports_color()
):
try:
from colorlog import ColoredFormatter # noqa: PLC0415
except ImportError:
pass
else:
colorizer = ColoredFormatter("%(log_color)s%(message)s")
return _ColoredFormatter(formatter, colorizer)
return formatter
def _get_handler(settings: Settings) -> logging.Handler:
"""Return a log handler object according to settings"""
filename = settings.get("LOG_FILE")
handler: logging.Handler
if filename:
mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
encoding = settings.get("LOG_ENCODING")
handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
elif settings.getbool("LOG_ENABLED"):
handler = logging.StreamHandler()
else:
handler = logging.NullHandler()
handler.setFormatter(_get_formatter(handler, settings))
handler.setLevel(settings.get("LOG_LEVEL"))
handler.addFilter(_SpiderRecordDefaultFilter())
if settings.getbool("LOG_SHORT_NAMES"):
handler.addFilter(TopLevelFormatter(["scrapy"]))
return handler
def log_scrapy_info(settings: Settings) -> None:
logger.info(
"Scrapy %(version)s started (bot: %(bot)s)",
{"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
)
software: list[str] = settings.getlist("LOG_VERSIONS")
if not software:
return
versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)
logger.info(f"Versions:\n{versions}")
def log_reactor_info() -> None:
from twisted.internet import reactor
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
logger.debug(
"Using asyncio event loop: %s.%s",
reactor._asyncioEventloop.__module__,
reactor._asyncioEventloop.__class__.__name__,
)
class StreamLogger:
"""Fake file-like stream object that redirects writes to a logger instance
Taken from:
https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
"""
def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):
self.logger: logging.Logger = logger
self.log_level: int = log_level
self.linebuf: str = ""
def write(self, buf: str) -> None:
for line in buf.rstrip().splitlines():
self.logger.log(self.log_level, line.rstrip())
def flush(self) -> None:
for h in self.logger.handlers:
h.flush()
class LogCounterHandler(logging.Handler):
"""Record log levels count into a crawler stats"""
def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.crawler: Crawler = crawler
def emit(self, record: logging.LogRecord) -> None:
sname = f"log_count/{record.levelname}"
self.crawler.stats.inc_value(sname)
_MSG_MAPPING_PLACEHOLDER = re.compile(r"%\(\w+\)")
def logformatter_adapter(
logkws: LogFormatterResult,
) -> tuple[Any, ...]:
"""
Helper that takes the dictionary output from the methods in LogFormatter
and adapts it into a tuple of positional arguments for logger.log calls.
"""
level = logkws.get("level", logging.INFO)
message = logkws.get("msg") or ""
args = logkws.get("args")
# logging interpolates the message whenever it receives any positional
# argument, so empty args are left out. Tuple args become one positional
# argument each, while a dict is a single positional argument.
if not args:
if _MSG_MAPPING_PLACEHOLDER.search(message):
# The log formatter method has already returned, so there is no
# frame of it left in the stack to point at. msg is part of the
# warning message instead, so that each offending method gets its
# own warning.
warnings.warn(
f"A log formatter method returned msg {message!r} with "
f"%(name)s placeholders and no args. Interpolating msg with "
f"the returned dict is deprecated, return those values under "
f"args instead.",
ScrapyDeprecationWarning,
stacklevel=1,
)
return (level, message, logkws)
return (level, message)
if isinstance(args, tuple):
return (level, message, *args)
return (level, message, args)
class SpiderLoggerAdapter(logging.LoggerAdapter[logging.Logger]):
def process(
self, msg: str, kwargs: MutableMapping[str, Any]
) -> tuple[str, MutableMapping[str, Any]]:
"""Method that augments logging with additional 'extra' data"""
if isinstance(kwargs.get("extra"), MutableMapping):
kwargs["extra"].update(self.extra)
else:
kwargs["extra"] = self.extra
return msg, kwargs