From dfe7541ef4b17081ce5c7760f90a89afb7554759 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sun, 15 Nov 2026 15:09:50 +0000 Subject: [PATCH] keep statistics and sessions between `run()` calls --- docs/03_concepts/22_statistics.mdx | 10 +-- docs/04_guides/11_secure_scraping.mdx | 2 +- docs/06_upgrading/upgrading_to_v2.md | 11 +-- ..._adaptive_playwright_crawler_statistics.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 23 ++----- src/crawlee/statistics/_models.py | 4 +- src/crawlee/statistics/_statistics.py | 3 + .../test_adaptive_playwright_crawler.py | 13 ++++ tests/unit/_statistics/test_enter_exit.py | 14 ++++ .../crawlers/_basic/test_basic_crawler.py | 68 +++++++++++++++---- 10 files changed, 107 insertions(+), 43 deletions(-) diff --git a/docs/03_concepts/22_statistics.mdx b/docs/03_concepts/22_statistics.mdx index 3a43ab16cc..c7f4eaf77c 100644 --- a/docs/03_concepts/22_statistics.mdx +++ b/docs/03_concepts/22_statistics.mdx @@ -20,7 +20,7 @@ The raw data lives in a `StatisticsState``Statistics.calculate` derives a `FinalStatistics` object. It's a frozen summary with the request counts, the retry histogram as a list, average finished and failed durations, total duration, per-minute rates for finished and failed requests, and the crawler runtime. The same summary appears in the periodic and final statistics logs. @@ -28,7 +28,7 @@ From the state, `Statistics.calculate`< There are two ways to get the statistics of a crawl: -- `crawler.run()` returns the `FinalStatistics` of the run once it ends. +- `crawler.run()` returns the `FinalStatistics` once the run ends. - `crawler.statistics` gives you the `Statistics` instance, so you can read the full `state` or call `calculate()` at any time, including from a request handler while the crawl is running. The following example reads the summary returned by `run()`, then looks at state fields that `FinalStatistics` doesn't include, and the errors collected during the run. @@ -37,7 +37,7 @@ The following example reads the summary returned by `run()`, then looks at state {ReadFinalStatisticsExample} -A crawler that you run again with `run()` after a completed run resets its statistics at the start of the new run. To keep the numbers from a previous run, store the returned `FinalStatistics` before calling `run()` again. +A crawler that you run again with `run()` continues the crawl, so its statistics add up the runs. The runtime counts only the time the runs took, not the time between them. The `FinalStatistics` that a later `run()` returns include the earlier runs too. To count from zero again, call `crawler.statistics.reset()` before the next `run()`. ## Periodic logging @@ -50,7 +50,7 @@ The crawler's constructor also accepts `statistics_log_format`. When the crawler ## Persistence -`Statistics` can save its state to a key-value store, so a crawl that gets migrated, aborted, or restarted continues counting from where it left off. When persistence is on, the state is loaded from the store when the crawl starts, saved on every `PERSIST_STATE` event, and saved once more when the crawl ends. For details about the event, see [Events](./events). +`Statistics` can save its state to a key-value store, so a crawl that gets migrated, aborted, or restarted continues counting from where it left off. When persistence is on, the state is loaded from the store when the crawl starts, saved on every `PERSIST_STATE` event, and saved once more when a run ends. For details about the event, see [Events](./events). The following options control persistence: @@ -74,6 +74,6 @@ A custom `Statistics` logs its periodic messages through the `crawlee.statistics ## Error tracking -Each `Statistics` instance holds an `error_tracker`, which groups errors from failed request attempts by their type, the place in the code where they were raised, and a similar message. Its `total` and `unique_error_count` properties count all errors and the distinct groups, and `get_most_common_errors()` returns the most frequent groups with their counts. At the end of a run that had errors, the crawler logs both counts. A second tracker, `error_tracker_retry`, collects session errors that caused a retry with a new session. +Each `Statistics` instance holds an `error_tracker`, which groups errors from failed request attempts by their type, the place in the code where they were raised, and a similar message. Its `total` and `unique_error_count` properties count all errors and the distinct groups, and `get_most_common_errors()` returns the most frequent groups with their counts. At the end of a run, the crawler logs both counts if there are any errors. A repeated `run()` keeps the errors of the earlier runs in the tracker, so the counts include them too. A second tracker, `error_tracker_retry`, collects session errors that caused a retry with a new session. With `save_error_snapshots=True`, the error tracker also saves a snapshot of the page on the first occurrence of each unique error. For details and examples, see [Capturing page snapshots on errors](./error-handling#capturing-page-snapshots-on-errors). diff --git a/docs/04_guides/11_secure_scraping.mdx b/docs/04_guides/11_secure_scraping.mdx index 253bdd5123..6d7a7b3fb2 100644 --- a/docs/04_guides/11_secure_scraping.mdx +++ b/docs/04_guides/11_secure_scraping.mdx @@ -62,7 +62,7 @@ If only your crawler's storage sees the response, the same request is a *blind* A hostile site can try to use up your CPU, memory, or queue rather than read anything: -- A *crawler trap* is a maze of randomly generated URLs linking to more generated pages. It's built to keep a crawler running forever and grow its queue without bound. Bound how far link-following goes with `max_crawl_depth`, and cap a whole run with `max_requests_per_crawl`. Both are unlimited by default, so set them yourself. +- A *crawler trap* is a maze of randomly generated URLs linking to more generated pages. It's built to keep a crawler running forever and grow its queue without bound. Bound how far link-following goes with `max_crawl_depth`, and cap a whole crawl with `max_requests_per_crawl`. Both are unlimited by default, so set them yourself. - A single *oversized or slow response* can tie up a worker or blow up memory. Both the fetch and the handler are capped at 60 seconds by default. `navigation_timeout` covers the fetch, including reading the body. `request_handler_timeout` covers the handler. With `FileDownloadCrawler` in stream mode, reading the body counts towards the handler timeout instead. Raise them if a legitimate target needs longer, but keep the cap as tight as your workload allows. - A *decompression bomb* is an archive that expands to gigabytes when unpacked. If your crawler downloads and extracts archives, cap the output stream during extraction. The declared size is part of the attack and easy to forge. diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 82609c9d68..bf002f18b2 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -267,11 +267,13 @@ crawler_1 = BasicCrawler(request_manager=queue) crawler_2 = BasicCrawler(request_manager=queue) ``` -## Repeated `run()` calls no longer purge the request queue +## Repeated `run()` calls continue the crawl In v1, calling `crawler.run()` again on the same instance purged the request queue, unless the queue was named or the previous run had failed. In v2, nothing is purged between runs, the same as Crawlee JS. A repeated `run()` continues with the same request manager and skips the requests an earlier run handled, failed ones included. -The `purge_request_queue` argument of `run()` is removed. If you passed `purge_request_queue=False`, drop the argument. To crawl the same requests again, purge the request manager yourself: +The statistics and the session pool carry over too. A repeated `run()` adds to the statistics of the earlier runs, error trackers included, and keeps the sessions. In v1, it started both afresh. The `max_requests_per_crawl` limit counts the requests in the statistics, so it includes the earlier runs. A run that starts with the limit already reached stops right away. To count from zero, call `crawler.statistics.reset()` before the next `run()`. To start with fresh sessions, pass your own `SessionPool` and call its `reset()` between runs. + +The `purge_request_queue` argument of `run()` is removed. If you passed `purge_request_queue=False`, drop the argument. To crawl the same requests again, purge the request manager yourself. To also count from zero, as v1 did, reset the statistics: ```python # v1 @@ -282,6 +284,7 @@ await crawler.run(urls) await crawler.run(urls) request_manager = await crawler.get_request_manager() await request_manager.purge() +crawler.statistics.reset() await crawler.run(urls) ``` @@ -501,7 +504,7 @@ What you do with an object you pass depends on its type: - `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished. - The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`. -- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time. +- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. Two crawlers can't run with the same instance at the same time. For example: @@ -518,8 +521,6 @@ async with session_pool: await crawler.run(urls) ``` -When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too. - `RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`. `AdaptivePlaywrightCrawler.rendering_type_predictor` is now read-only. Pass your predictor to the constructor instead. diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py index 5153373235..39abc52de7 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler_statistics.py @@ -11,7 +11,7 @@ @docs_group('Statistics') class AdaptivePlaywrightCrawlerStatisticState(StatisticsState): - """Statistic data about a crawler run with additional information related to adaptive crawling.""" + """Statistic data about a crawl with additional information related to adaptive crawling.""" model_config = ConfigDict( alias_generator=to_camel, validate_by_name=True, validate_by_alias=True, ser_json_inf_nan='constants' diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 494d42d0ba..9984231bff 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -156,7 +156,8 @@ class _BasicCrawlerOptions(TypedDict): max_requests_per_crawl: NotRequired[int | None] """Maximum number of pages to open during a crawl. The crawl stops upon reaching this limit. Setting this value can help avoid infinite loops in misconfigured crawlers. `None` means no limit. - Due to concurrency settings, the actual number of pages visited may slightly exceed this value.""" + Due to concurrency settings, the actual number of pages visited may slightly exceed this value. The limit counts + the requests in the statistics, so it includes earlier runs and other crawlers that share the statistics.""" max_session_rotations: NotRequired[int] """Maximum number of session rotations per request. The crawler rotates the session if a proxy error occurs @@ -348,7 +349,8 @@ def __init__( this limit. Setting this value can help avoid infinite loops in misconfigured crawlers. `None` means no limit. Due to concurrency settings, the actual number of pages visited may slightly exceed this value. If used together with `keep_alive`, then the crawler will be kept alive only until - `max_requests_per_crawl` is achieved. + `max_requests_per_crawl` is achieved. The limit counts the requests in the statistics, so it includes + earlier runs and other crawlers that share the statistics. max_session_rotations: Maximum number of session rotations per request. The crawler rotates the session if a proxy error occurs or if the website blocks the request. The session rotations are not counted towards the `max_request_retries` limit. @@ -534,7 +536,6 @@ async def persist_state_factory() -> KeyValueStore: # State flags self._keep_alive = keep_alive - self._has_finished_before = False self._stop_before_run = False self._logger_once = LoggerOnce(self._logger) @@ -598,7 +599,7 @@ def concurrency_system(self) -> ConcurrencySystemInterface | None: @property def statistics(self) -> Statistics[TStatisticsState]: - """Statistics about the current (or last) crawler run.""" + """Statistics of the crawler's runs, which add up from one run to the next.""" return self._statistics @property @@ -837,7 +838,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta Calling `run()` again on the same instance keeps crawling the same request manager. Requests that a previous run handled, failed ones included, aren't processed again. To crawl them again, purge the request manager, or - create a new crawler with a freshly opened request queue. + create a new crawler with a freshly opened request queue. The statistics carry over too, so + `max_requests_per_crawl` includes the requests of the earlier runs. Args: requests: The requests to be enqueued before the crawler starts. @@ -869,15 +871,6 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta self._last_run = self._create_run() try: - if self._has_finished_before: - self._statistics.reset() - await self._statistics.reset_store() - - # A session pool passed in belongs to the caller, who may still use it in another crawler. - if self._session_pool in self._owned_collaborators: - self._session_pool.reset() - await self._session_pool.reset_store() - if requests is not None: await self.add_requests(requests) @@ -937,8 +930,6 @@ def sigint_handler() -> None: 'The crawl was interrupted. To resume, do: CRAWLEE_PURGE_ON_START=0 python %s', sys.argv[0] ) - self._has_finished_before = True - await self._save_crawler_state() final_statistics = self._statistics.calculate() diff --git a/src/crawlee/statistics/_models.py b/src/crawlee/statistics/_models.py index 8a7d2a3873..b6f5682f13 100644 --- a/src/crawlee/statistics/_models.py +++ b/src/crawlee/statistics/_models.py @@ -20,7 +20,7 @@ @dataclass(frozen=True) @docs_group('Statistics') class FinalStatistics: - """Statistics about a crawler run.""" + """Statistics about a crawl.""" requests_finished: int requests_failed: int @@ -56,7 +56,7 @@ def __str__(self) -> str: @docs_group('Statistics') class StatisticsState(BaseModel): - """Statistic data about a crawler run.""" + """Statistic data about a crawl.""" model_config = ConfigDict( alias_generator=to_camel, validate_by_name=True, validate_by_alias=True, ser_json_inf_nan='constants' diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py index 3c684a4da1..9fbed37a27 100644 --- a/src/crawlee/statistics/_statistics.py +++ b/src/crawlee/statistics/_statistics.py @@ -174,6 +174,9 @@ async def __aenter__(self) -> Self: self._active = True + # An attempt that an earlier run left in progress never finishes, so it must not count as a retry in this run. + self._requests_in_progress.clear() + try: await self._state.initialize() diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 2db1cf4574..aae9b4c288 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -736,6 +736,19 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_browser_handler.assert_called_once_with() +async def test_repeated_run_keeps_own_stats(test_urls: list[str]) -> None: + """A repeated `run()` adds to the default statistics of the crawler, since it continues the crawl.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0])), + ) + crawler.router.default_handler(AsyncMock()) + + await crawler.run(test_urls[:1]) + stats = await crawler.run(test_urls[1:]) + + assert stats.requests_finished == len(test_urls) + + async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None: """Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant. diff --git a/tests/unit/_statistics/test_enter_exit.py b/tests/unit/_statistics/test_enter_exit.py index 2324af6941..f8dc44d61f 100644 --- a/tests/unit/_statistics/test_enter_exit.py +++ b/tests/unit/_statistics/test_enter_exit.py @@ -68,6 +68,20 @@ async def wait_for_release() -> None: await exit_task +async def test_enter_drops_unfinished_requests() -> None: + """A request left in progress by an earlier run doesn't count as retried in the next one.""" + statistics = Statistics.with_default_state() + + async with statistics: + statistics.record_request_processing_start('request') + + async with statistics: + statistics.record_request_processing_start('request') + statistics.record_request_processing_finish('request') + + assert statistics.state.request_retry_histogram == {0: 1} + + async def test_failed_start_tears_down_stats(monkeypatch: pytest.MonkeyPatch) -> None: """Statistics whose entry fails after loading the state tear the state down and can be entered again.""" statistics = Statistics.with_default_state() diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 3e7f93330f..2a02f1be12 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -1467,15 +1467,57 @@ async def handler(context: BasicCrawlingContext) -> None: assert handled_urls == ['https://a.placeholder.com', 'https://b.placeholder.com'] -async def test_repeated_run_starts_with_fresh_statistics() -> None: - """A repeated `run()` starts from fresh statistics instead of adding to the previous run's.""" +async def test_repeated_run_keeps_own_stats() -> None: + """A repeated `run()` adds to the statistics the crawler built, since it continues the crawl.""" crawler = BasicCrawler(request_handler=AsyncMock()) await crawler.run(['https://a.placeholder.com']) stats = await crawler.run(['https://b.placeholder.com']) + assert stats.requests_finished == 2 + + +async def test_request_limit_spans_runs() -> None: + """`max_requests_per_crawl` counts the earlier runs, so a run that starts at the limit handles no request.""" + handler = AsyncMock() + crawler = BasicCrawler(request_handler=handler, max_requests_per_crawl=1) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + handler.assert_awaited_once() + + +async def test_repeated_run_keeps_stats() -> None: + """A repeated `run()` adds to statistics passed to the crawler.""" + crawler = BasicCrawler(request_handler=AsyncMock(), statistics=Statistics.with_default_state()) + + await crawler.run(['https://a.placeholder.com']) + stats = await crawler.run(['https://b.placeholder.com']) + + assert stats.requests_finished == 2 + + +async def test_repeated_run_keeps_errors() -> None: + """A repeated `run()` adds to the errors that the error tracker collected in the earlier runs.""" + crawler = BasicCrawler(max_request_retries=0) + crawler.router.default_handler(AsyncMock(side_effect=RuntimeError('Handler failed'))) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + assert crawler.statistics.error_tracker.total == 2 + + +async def test_stats_reset_between_runs() -> None: + """Statistics reset between runs make the next run count from zero, although the crawler persists them.""" + crawler = BasicCrawler(request_handler=AsyncMock()) + + await crawler.run(['https://a.placeholder.com']) + crawler.statistics.reset() + stats = await crawler.run(['https://b.placeholder.com']) + assert stats.requests_finished == 1 - assert crawler.statistics.state.requests_finished == 1 async def test_crawlers_share_stats_in_turn() -> None: @@ -1587,20 +1629,20 @@ async def test_repeated_run_keeps_sessions() -> None: assert session_pool.active -async def test_repeated_run_resets_own_sessions() -> None: - """A repeated `run()` replaces the sessions of the session pool the crawler built.""" +async def test_repeated_run_keeps_own_pool() -> None: + """A repeated `run()` keeps the sessions of the session pool the crawler built, since it continues the crawl.""" crawler = BasicCrawler() - session_ids = list[set[str]]() + pool_session_ids = list[set[str]]() @crawler.router.default_handler async def handler(_context: BasicCrawlingContext) -> None: - session_ids.append(set(crawler._session_pool.get_state(as_dict=False).sessions)) + pool_session_ids.append(set(crawler._session_pool.get_state(as_dict=False).sessions)) await crawler.run(['https://a.placeholder.com']) await crawler.run(['https://b.placeholder.com']) - first_ids, second_ids = session_ids - assert first_ids.isdisjoint(second_ids) + first_run_ids, second_run_ids = pool_session_ids + assert first_run_ids <= second_run_ids async def test_run_needs_entered_session_pool() -> None: @@ -1847,14 +1889,14 @@ async def test_handled_count_error_does_not_fail_the_run( queue = await RequestQueue.open() crawler = BasicCrawler(request_manager=queue) crawler.router.default_handler(AsyncMock()) - await crawler.run(['https://a.placeholder.com']) + first_stats = await crawler.run(['https://a.placeholder.com']) get_handled_count = AsyncMock(side_effect=RuntimeError('Handled count unavailable')) monkeypatch.setattr(queue, 'get_handled_count', get_handled_count) with caplog.at_level(logging.WARNING): stats = await crawler.run(['https://a.placeholder.com']) - assert stats.requests_total == 0 + assert stats.requests_total == first_stats.requests_total get_handled_count.assert_awaited_once() assert processed_no_requests_warnings(caplog) == [] @@ -1897,12 +1939,12 @@ async def test_aborted_crawl_with_pending_requests_does_not_warn(caplog: pytest. ) handler = AsyncMock(side_effect=[None, RuntimeError('Handler failed')]) crawler.router.default_handler(handler) - await crawler.run(['https://a.placeholder.com']) + first_stats = await crawler.run(['https://a.placeholder.com']) with caplog.at_level(logging.WARNING): stats = await crawler.run(['https://b.placeholder.com']) - assert stats.requests_total == 0 + assert stats.requests_total == first_stats.requests_total assert processed_no_requests_warnings(caplog) == []