Source code for scrapy.extensions.memdebug

"""
MemoryDebugger extension

See documentation in docs/topics/extensions.rst
"""

import gc

from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.trackref import live_refs


[docs]class MemoryDebugger: def __init__(self, stats): self.stats = stats @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o def spider_closed(self, spider, reason): gc.collect() self.stats.set_value( "memdebug/gc_garbage_count", len(gc.garbage), spider=spider ) for cls, wdict in live_refs.items(): if not wdict: continue self.stats.set_value( f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider )