"""
Depth Spider Middleware
See documentation in docs/topics/spider-middleware.rst
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.statscollectors import StatsCollector
logger = logging.getLogger(__name__)
[docs]
class DepthMiddleware(BaseSpiderMiddleware):
"""Track the depth of each request within the site being scraped, setting
``request.meta["depth"]`` to 0 when there is no value previously set
(usually just the first request) and incrementing it by 1 otherwise.
It can be used to limit the maximum depth to scrape, control request
priority based on their depth, and things like that, through the
:setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and
:setting:`DEPTH_PRIORITY` settings.
.. reqmeta:: depth_reset
depth_reset
-----------
.. versionadded:: 2.18.0
:attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request
depth 0 instead of the depth of its source response plus 1, e.g. to keep
:setting:`DEPTH_LIMIT` from applying across a domain change.
"""
crawler: Crawler
def __init__( # pylint: disable=super-init-not-called
self,
maxdepth: int,
stats: StatsCollector,
verbose_stats: bool = False,
prio: int = 1,
):
self.maxdepth = maxdepth
self.stats = stats
self.verbose_stats = verbose_stats
self.prio = prio
self._ignored_logged = False
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
settings = crawler.settings
maxdepth = settings.getint("DEPTH_LIMIT")
verbose = settings.getbool("DEPTH_STATS_VERBOSE")
prio = settings.getint("DEPTH_PRIORITY")
o = cls(maxdepth, crawler.stats, verbose, prio)
o.crawler = crawler
return o
@_warn_spider_arg
def process_spider_output(
self,
response: Response | None,
result: Iterable[Any],
spider: Spider | None = None,
) -> Iterable[Any]:
if response is not None:
self._init_depth(response)
yield from super().process_spider_output(response, result)
@_warn_spider_arg
async def process_spider_output_async(
self,
response: Response | None,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
if response is not None:
self._init_depth(response)
async for o in super().process_spider_output_async(response, result):
yield o
def _init_depth(self, response: Response) -> None:
# base case (depth=0)
if "depth" not in response.meta:
response.meta["depth"] = 0
if self.verbose_stats:
self.stats.inc_value("request_depth_count/0")
def get_processed_request(
self, request: Request, response: Response | None
) -> Request | None:
# Consumed here so that it cannot reach response.meta and, from there,
# spread to further requests through a meta copy.
depth_reset = request.meta.pop("depth_reset", False)
if response is None:
# start requests
return request
depth = 0 if depth_reset else response.meta["depth"] + 1
request.meta["depth"] = depth
if self.prio:
request.priority -= depth * self.prio
if self.maxdepth and depth > self.maxdepth:
if not self._ignored_logged:
logger.debug(
f"Ignoring link (depth > {self.maxdepth}): {request.url}"
" - no more ignored links will be shown",
extra={"spider": self.crawler.spider},
)
self._ignored_logged = True
self.stats.inc_value("depth/request_ignored_count")
return None
if self.verbose_stats:
self.stats.inc_value(f"request_depth_count/{depth}")
self.stats.max_value("request_depth_max", depth)
return request