diff --git a/src/apify_client/_streamed_log.py b/src/apify_client/_streamed_log.py index f458c8e3..687a3215 100644 --- a/src/apify_client/_streamed_log.py +++ b/src/apify_client/_streamed_log.py @@ -19,6 +19,10 @@ from apify_client.types import Timeout +_ANSI_CODE = re.compile(r'\x1b\[[0-9;]*m') +_LEVEL_PATTERN = re.compile(r'\b(CRITICAL|FATAL|ERROR|WARNING|WARN|INFO|DEBUG)\b') + + class StreamedLogBase: """Base class for streaming and buffering chunked Actor run logs.""" @@ -97,15 +101,9 @@ def _process_whole_log(self, log: bytes | None) -> None: @staticmethod def _guess_log_level_from_message(message: str) -> int: - """Guess the log level from the message.""" - # Using only levels explicitly mentioned in the logging module - known_levels = ('CRITICAL', 'FATAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET') - level_names_to_levels = logging.getLevelNamesMapping() - for level in known_levels: - if level in message: - return level_names_to_levels[level] - # Unknown log level. Fall back to the default. - return logging.INFO + """Return the first whole-word level name found in the message, ignoring ANSI colors. Default to INFO.""" + match = _LEVEL_PATTERN.search(_ANSI_CODE.sub('', message)) + return logging.getLevelNamesMapping()[match[1]] if match else logging.INFO @docs_group('Other') @@ -113,7 +111,7 @@ class StreamedLog(StreamedLogBase): """Streams Actor run log output to a Python logger in a background thread. The log stream is consumed in a background thread and each log message is forwarded to the provided logger with - an appropriate log level inferred from the message content. + the first level name found in the message, defaulting to INFO when there is none. Can be used as a context manager, which automatically starts and stops the streaming thread. Alternatively, call `start` and `stop` manually. Obtain an instance via `RunClient.get_streamed_log`. @@ -256,12 +254,15 @@ class StreamedLogAsync(StreamedLogBase): """Streams Actor run log output to a Python logger in an asyncio task. The log stream is consumed in a background asyncio task and each log message is forwarded to the provided logger - with an appropriate log level inferred from the message content. + with the first level name found in the message, defaulting to INFO when there is none. Can be used as an async context manager, which automatically starts and cancels the streaming task. Alternatively, call `start` and `stop` manually. Obtain an instance via `RunClientAsync.get_streamed_log`. """ + _stop_timeout_s: ClassVar[float] = 1 + """Grace period for in-flight log data before the streaming task is cancelled.""" + def __init__(self, log_client: LogClientAsync, *, to_logger: logging.Logger, from_start: bool = True) -> None: """Initialize `StreamedLogAsync`. @@ -288,19 +289,24 @@ def start(self) -> Task: return self._streaming_task async def stop(self) -> None: - """Stop the streaming task. + """Wait up to one second for the stream to end, then cancel the streaming task. + Caller cancellation cancels the streaming task and propagates without reading the whole log. If no stream has delivered anything yet, read the whole log in one request instead. """ if not self._streaming_task: raise RuntimeError('Streaming task is not active') was_streaming = not self._streaming_task.done() - self._streaming_task.cancel() try: - await self._streaming_task - except asyncio.CancelledError: + await asyncio.wait_for(self._streaming_task, timeout=self._stop_timeout_s) + except TimeoutError: pass + except asyncio.CancelledError: + # Swallow the task's own cancellation, but let a cancellation of the caller propagate. + current = asyncio.current_task() + if current is not None and current.cancelling(): + raise finally: self._streaming_task = None if not was_streaming or self._received_data: diff --git a/tests/unit/test_streamed_log.py b/tests/unit/test_streamed_log.py index f23ff9d0..87c109c5 100644 --- a/tests/unit/test_streamed_log.py +++ b/tests/unit/test_streamed_log.py @@ -1,11 +1,16 @@ from __future__ import annotations +import asyncio import logging -from unittest.mock import Mock +from typing import TYPE_CHECKING +from unittest.mock import AsyncMock, Mock import pytest -from apify_client._streamed_log import StreamedLogBase +from apify_client._streamed_log import StreamedLogAsync, StreamedLogBase + +if TYPE_CHECKING: + from collections.abc import AsyncIterator @pytest.mark.parametrize( @@ -38,3 +43,108 @@ def test_streamed_log_redirects_each_complete_line(chunks: list[bytes], expected calls = [(call.kwargs['level'], call.kwargs['msg']) for call in to_logger.log.call_args_list] assert calls == expected_calls + + +@pytest.mark.parametrize( + ('content', 'level'), + [ + pytest.param('INFO Saved ERROR_PAGE screenshot', logging.INFO, id='info-with-error-text'), + pytest.param('DEBUG WARNING in payload', logging.DEBUG, id='debug-with-warning-text'), + pytest.param('ERROR failed\nCRITICAL detail', logging.ERROR, id='multiline'), + pytest.param('[apify] DEBUG ERROR in payload', logging.DEBUG, id='logger-prefix'), + pytest.param('WARNING warning', logging.WARNING, id='warning'), + pytest.param('FATAL failed', logging.FATAL, id='fatal'), + pytest.param('ERROR_PAGE saved', logging.INFO, id='unknown-token'), + pytest.param('Saved ERROR screenshot', logging.ERROR, id='first-level-word'), + pytest.param('Saved screenshot', logging.INFO, id='no-level'), + pytest.param('\x1b[32mINFO\x1b[0m ready ERROR', logging.INFO, id='ansi-colors'), + pytest.param('', logging.INFO, id='empty'), + ], +) +def test_streamed_log_level_uses_token(content: str, level: int) -> None: + """The first whole-word level name, with ANSI colors removed, determines severity.""" + logger = Mock(spec=logging.Logger) + stream = StreamedLogBase(logger) + stream._process_whole_log(f'2026-11-15T13:00:00.000Z {content}'.encode()) + assert logger.log.call_args.kwargs['level'] == level + + +async def test_async_stop_drains_in_flight_chunk() -> None: + """Stopping lets the last in-flight chunk reach the logger before the stream ends.""" + delivered = asyncio.Event() + release = asyncio.Event() + + async def chunks() -> AsyncIterator[bytes]: + yield b'2026-11-15T13:00:00.000Z INFO first\n' + delivered.set() + await release.wait() + yield b'2026-11-15T13:00:01.000Z INFO last\n' + + response = Mock() + response.aiter_bytes.side_effect = chunks + context = AsyncMock() + context.__aenter__.return_value = response + client = Mock() + client.stream.return_value = context + logger = Mock(spec=logging.Logger) + stream = StreamedLogAsync(client, to_logger=logger) + task = stream.start() + await delivered.wait() + asyncio.get_running_loop().call_soon(release.set) + await stream.stop() + assert not task.cancelled() + assert len(logger.log.call_args_list) == 2 + assert logger.log.call_args.kwargs['msg'].endswith('INFO last') + + +async def test_async_stop_cancels_silent_stream(monkeypatch: pytest.MonkeyPatch) -> None: + """A stream that stays silent is cancelled after the grace period.""" + logger = Mock(spec=logging.Logger) + stream = StreamedLogAsync(Mock(), to_logger=logger) + monkeypatch.setattr(StreamedLogAsync, '_stop_timeout_s', 0.01, raising=False) + stream._received_data = True + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + await stream.stop() + assert task.cancelled() + assert stream._streaming_task is None + + +async def test_async_stop_cancels_stream_when_caller_is_cancelled() -> None: + """Cancelling stop during the grace period must not leave it waiting for the stream.""" + stream = StreamedLogAsync(Mock(), to_logger=Mock(spec=logging.Logger)) + stream._log_client.get_as_bytes = AsyncMock() + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + stopper = asyncio.create_task(stream.stop()) + await asyncio.sleep(0) + stopper.cancel() + try: + done, _ = await asyncio.wait({stopper}, timeout=0.1) + assert stopper in done + assert stopper.cancelled() + stream._log_client.get_as_bytes.assert_not_awaited() + assert task.cancelled() + assert stream._streaming_task is None + finally: + task.cancel() + await asyncio.gather(task, stopper, return_exceptions=True) + + +async def test_async_stop_propagates_caller_timeout() -> None: + """A caller timeout cancels the stream and skips the whole-log fallback.""" + client = Mock() + client.get_as_bytes = AsyncMock() + stream = StreamedLogAsync(client, to_logger=Mock(spec=logging.Logger)) + task = asyncio.create_task(asyncio.Event().wait()) + stream._streaming_task = task + try: + with pytest.raises(TimeoutError): + async with asyncio.timeout(0.01): + await stream.stop() + assert task.cancelled() + assert stream._streaming_task is None + client.get_as_bytes.assert_not_awaited() + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True)