Source code for scrapy.pqueues

from __future__ import annotations

import hashlib
import logging
from contextlib import suppress
from pathlib import Path
from typing import TYPE_CHECKING, Any, Protocol, Self, cast

from scrapy.utils.misc import build_from_crawler

if TYPE_CHECKING:
    from collections.abc import Iterable

    from scrapy import Request
    from scrapy.core.downloader import Downloader
    from scrapy.crawler import Crawler

logger = logging.getLogger(__name__)


def _path_safe(text: str) -> str:
    """
    Return a filesystem-safe version of a string ``text``

    >>> _path_safe('simple.org').startswith('simple.org')
    True
    >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
    True
    >>> _path_safe('some@symbol?').startswith('some_symbol_')
    True
    """
    pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
    # Replacing characters can make different inputs collapse to the same
    # prefix, so a hash of the original text keeps the result unique.
    unique_slot = hashlib.md5(text.encode()).hexdigest()  # noqa: S324
    return f"{pathable_slot}-{unique_slot}"


class QueueProtocol(Protocol):
    """Protocol for downstream queues of ``ScrapyPriorityQueue``."""

    def push(self, request: Request) -> None: ...

    def pop(self) -> Request | None: ...

    def close(self) -> None: ...

    def __len__(self) -> int: ...


[docs] class PriorityQueueProtocol(Protocol): """Interface of :setting:`SCHEDULER_PRIORITY_QUEUE` classes. :attr:`changed`, :meth:`next_priority` and :meth:`state` are optional. """
[docs] @classmethod def from_crawler( cls, crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, startprios: Any = (), *, start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: """Return a priority queue that stores requests in instances of *downstream_queue_cls*, or of *start_queue_cls* for :ref:`start requests <start-requests>` if it is not ``None``. *key* is the directory where the queue persists its requests, or an empty string for an in-memory queue. *startprios* is what :meth:`close` returned the last time the job stopped. """
[docs] def push(self, request: Request) -> None: """Store *request*."""
[docs] def pop(self) -> Request | None: """Remove and return the next request, or return ``None`` if the queue is empty. """
[docs] def close(self) -> Any: """Close the queue and return a JSON-serializable state to pass as *startprios* to :meth:`from_crawler` when resuming the job. """
[docs] def __len__(self) -> int: """Return the number of stored requests."""
changed: bool """Whether the return value of :meth:`state` changed since this attribute was last set to ``False``. With :setting:`JOBDIR_SYNC_EVERY`, the scheduler persists the state while the job runs, and resets this attribute after persisting it. """
[docs] def next_priority(self) -> int | None: """Return the priority of the request that :meth:`pop` would return next, or ``None`` if the queue is empty. The scheduler pops requests from whichever of its memory and disk queues returns the lowest value, or from its memory queue on a tie, so any scale works as long as lower values pop first. Without this method, the scheduler pops memory requests first. """
[docs] def state(self) -> Any: """Return the same as :meth:`close`, without closing the queue."""
[docs] class ScrapyPriorityQueue: """Priority queue that pops requests from highest to lowest :attr:`~scrapy.Request.priority`, in :ref:`request order <request-order>` within a priority. Disk persistence ================ .. warning:: The files that this class generates on disk are an implementation detail, and may change without a warning in a future version of Scrapy. Do not rely on the following information for anything other than debugging purposes. When a component instantiates this class with a non-empty *key* argument, *key* is used as a persistence directory. For every request enqueued, this class checks: - Whether the request is a :ref:`start request <start-requests>` or not. - The :data:`~scrapy.Request.priority` of the request. For each combination of the above seen, this class creates an instance of *downstream_queue_cls* (or *start_queue_cls* for start requests if it was passed) with *key* set to a subdirectory of the persistence directory, named as the negated request priority (e.g. ``-1``), with an ``s`` suffix in case of a start request (e.g. ``-1s``). """ @classmethod def from_crawler( cls, crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), *, start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: return cls( crawler, downstream_queue_cls, key, startprios, start_queue_cls=start_queue_cls, ) def __init__( self, crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, startprios: Iterable[int] = (), *, start_queue_cls: type[QueueProtocol] | None = None, ): self.crawler: Crawler = crawler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.queues: dict[int, QueueProtocol] = {} self._start_queues: dict[int, QueueProtocol] = {} self.curprio: int | None = None self.changed: bool = False self.init_prios(startprios) def init_prios(self, startprios: Iterable[int]) -> None: if not startprios: return for priority in startprios: q = self.qfactory(priority) if q: self.queues[priority] = q else: q.close() self.changed = True if self._start_queue_cls: q = self._sqfactory(priority) if q: self._start_queues[priority] = q else: q.close() self.changed = True # A recorded priority may have no queue to restore, e.g. if it only # ever held a request that failed to serialize, so curprio comes from # the queues that do exist and not from startprios. self._update_curprio() def qfactory(self, key: int) -> QueueProtocol: return build_from_crawler( self.downstream_queue_cls, self.crawler, self.key + "/" + str(key), ) def _sqfactory(self, key: int) -> QueueProtocol: assert self._start_queue_cls is not None return build_from_crawler( self._start_queue_cls, self.crawler, f"{self.key}/{key}s", ) def priority(self, request: Request) -> int: return -request.priority def push(self, request: Request) -> None: priority = self.priority(request) is_start_request = request.meta.get("is_start_request", False) if is_start_request and self._start_queue_cls: if priority not in self._start_queues: self._start_queues[priority] = self._sqfactory(priority) self.changed = True q = self._start_queues[priority] else: if priority not in self.queues: self.queues[priority] = self.qfactory(priority) self.changed = True q = self.queues[priority] q.push(request) # this may fail (eg. serialization error) # A queue class may drop a request instead of storing it, and a # priority with an empty queue must not become the current one, since # pop() and peek() expect to find a request there. if q and (self.curprio is None or priority < self.curprio): self.curprio = priority def _curqueue(self) -> QueueProtocol: assert self.curprio is not None # Whichever dict holds a non-empty queue at the current priority. The # other one may hold an empty queue left behind by a failed push, e.g. # on a serialization error, which would hide the next request. return self.queues.get(self.curprio) or self._start_queues[self.curprio] def pop(self) -> Request | None: if self.curprio is None: return None q = self._curqueue() request = q.pop() if not q: self._update_curprio() return request def _update_curprio(self) -> None: # Keeping an empty queue would hold its storage open for nothing, and # make close() record a priority with nothing to restore from it. prios: set[int] = set() for queues in (self.queues, self._start_queues): for p, q in list(queues.items()): if q: prios.add(p) else: del queues[p] q.close() self.changed = True self.curprio = min(prios) if prios else None def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. Raises :exc:`NotImplementedError` if the underlying queue class does not implement a ``peek`` method, which is optional for queues. """ if self.curprio is None: return None # Picking the queue the same way as pop() is what makes the returned # request the one that pop() then returns. queue = self._curqueue() # Protocols can't declare optional members return cast("Request", queue.peek()) # type: ignore[attr-defined] def next_priority(self) -> int | None: return self.curprio def state(self) -> list[int]: return sorted(self.queues.keys() | self._start_queues.keys()) def close(self) -> list[int]: active = self.state() for queues in (self.queues, self._start_queues): for q in queues.values(): q.close() return active def __len__(self) -> int: return ( sum( len(x) for queues in (self.queues, self._start_queues) for x in queues.values() ) if self.queues or self._start_queues else 0 )
class DownloaderInterface: def __init__(self, crawler: Crawler): self.downloader: Downloader = crawler.engine.downloader def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: return [(self._active_downloads(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: return self.downloader.get_slot_key(request) def _active_downloads(self, slot: str) -> int: """Return a number of requests in a Downloader for a given slot""" if slot not in self.downloader.slots: return 0 return len(self.downloader.slots[slot].active)
[docs] class DownloaderAwarePriorityQueue: """PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. Disk persistence ================ .. warning:: The files that this class generates on disk are an implementation detail, and may change without a warning in a future version of Scrapy. Do not rely on the following information for anything other than debugging purposes. When a component instantiates this class with a non-empty *key* argument, *key* is used as a persistence directory, and inside that directory this class creates a subdirectory per download slot (domain). Those subdirectories are named after the corresponding download slot, with path-unsafe characters replaced by underscores and an MD5 hash suffix to avoid collisions. For each download slot, this class creates an instance of :class:`ScrapyPriorityQueue` with the download slot subdirectory as *key* and its own *downstream_queue_cls*. """ @classmethod def from_crawler( cls, crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, startprios: dict[str, Iterable[int]] | None = None, *, start_queue_cls: type[QueueProtocol] | None = None, ) -> Self: return cls( crawler, downstream_queue_cls, key, startprios, start_queue_cls=start_queue_cls, ) def __init__( self, crawler: Crawler, downstream_queue_cls: type[QueueProtocol], key: str, slot_startprios: dict[str, Iterable[int]] | None = None, *, start_queue_cls: type[QueueProtocol] | None = None, ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' ) if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError( "DownloaderAwarePriorityQueue accepts " "``slot_startprios`` as a dict; " f"{slot_startprios.__class__!r} instance " "is passed. Most likely, it means the state is " "created by an incompatible priority queue. " "Only a crawl started with the same priority " "queue class can be resumed." ) self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key self.crawler: Crawler = crawler self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue self._last_selected_slot: str | None = None self.changed: bool = False if slot_startprios: for slot, startprios in slot_startprios.items(): self.pqueues[slot] = self.pqfactory(slot, startprios) self._collect_change(self.pqueues[slot]) def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str: # Lexicographic on (active, slot): the least-loaded slot, and the # lowest-named one among ties. min_active, slot = min(stats) last = self._last_selected_slot if last is not None: # Round-robin among the tied slots: the first one after the last # selected, so that equally-loaded slots take turns. slot = min( (s for active, s in stats if active == min_active and s > last), default=slot, ) if update_state: self._last_selected_slot = slot return slot def pqfactory( self, slot: str, startprios: Iterable[int] = () ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( self.crawler, self.downstream_queue_cls, self.key + "/" + _path_safe(slot), startprios, start_queue_cls=self._start_queue_cls, ) def _collect_change(self, queue: ScrapyPriorityQueue) -> None: if queue.changed: queue.changed = False self.changed = True def pop(self) -> Request | None: stats = self._downloader_interface.stats(self.pqueues) if not stats: return None slot = self._next_slot(stats, update_state=True) queue = self.pqueues[slot] request = queue.pop() self._collect_change(queue) if len(queue) == 0: del self.pqueues[slot] queue.close() self.changed = True if self.key: # Reclaim the slot directory; rmdir leaves it alone if the # downstream queues did not remove all their files. with suppress(OSError): Path(self.key, _path_safe(slot)).rmdir() return request def push(self, request: Request) -> None: slot = self._downloader_interface.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) self.changed = True queue = self.pqueues[slot] queue.push(request) self._collect_change(queue) def peek(self) -> Request | None: """Returns the next object to be returned by :meth:`pop`, but without removing it from the queue. Raises :exc:`NotImplementedError` if the underlying queue class does not implement a ``peek`` method, which is optional for queues. """ stats = self._downloader_interface.stats(self.pqueues) if not stats: return None slot = self._next_slot(stats, update_state=False) queue = self.pqueues[slot] return queue.peek() def next_priority(self) -> int | None: if not self.pqueues: return None stats = self._downloader_interface.stats(self.pqueues) slot = self._next_slot(stats, update_state=False) return self.pqueues[slot].next_priority() def state(self) -> dict[str, list[int]]: return {slot: queue.state() for slot, queue in self.pqueues.items()} def close(self) -> dict[str, list[int]]: active = self.state() for queue in self.pqueues.values(): queue.close() self.pqueues.clear() return active def __len__(self) -> int: return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 def __contains__(self, slot: str) -> bool: return slot in self.pqueues