adding support for log rotation per issue scrapy#3628

docs/topics/logging.rst

@@ -170,6 +170,12 @@ These settings can be used to configure the logging:
 * :setting:`LOG_DATEFORMAT`
 * :setting:`LOG_STDOUT`
 * :setting:`LOG_SHORT_NAMES`
+* :setting:`LOG_FILE_ROTATE`
+* :setting:`LOG_FILE_ROTATE_WHEN`
+* :setting:`LOG_FILE_ROTATE_INTERVAL`
+* :setting:`LOG_FILE_ROTATE_BACKEUP_COUNT`
+* :setting:`LOG_FILE_ROTATE_UTC`
+* :setting:`LOG_FILE_ROTATE_AT_TIME`
 
 The first couple of settings define a destination for log messages. If
 :setting:`LOG_FILE` is set, messages sent through the root logger will be
@@ -194,6 +200,33 @@ If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
 component that prints the log. It is unset by default, hence logs contain the
 Scrapy component responsible for that log output.
 
+When :setting:`LOG_FILE_ROTATE` is enabled, time-based log rotation is activated.
+This feature appends date-and-time information to log file extensions, formatted as
+%Y-%m-%d_%H-%M-%S or a leading portion based on the rollover interval.
+
+The :setting:`LOG_FILE_ROTATE_WHEN` setting determines the timing for log file
+replacement and rotation. Its default value is ``midnight``, but it can be set
+to various intervals:
+
+- ``S`` for seconds,
+- ``M`` for minutes,
+- ``H`` for hours,
+- ``D`` for days,
+- ``W0`` to ``W6`` for specific weekdays (in this case, the interval value is ignored).
+
+:setting:`LOG_FILE_ROTATE_INTERVAL` determines the frequency of the log file rotation.
+It specifies the interval at which the log files are rotated, depending on the unit set
+by :setting:`LOG_FILE_ROTATE_WHEN`. For instance, if `LOG_FILE_ROTATE_WHEN` is set to ``H``
+(hours), and `LOG_FILE_ROTATE_INTERVAL` is set to `6`, the log files will rotate every 6 hours.
+
+If :setting:`LOG_FILE_ROTATE_UTC` is true, UTC times are used for rotation; otherwise,
+local time is used. The :setting:`LOG_FILE_ROTATE_BACKUP_COUNT`, if nonzero, limits the
+number of backup files retained. Upon rollover, the oldest file is deleted if creating a
+new one would exceed this limit. If :setting:`LOG_FILE_ROTATE_AT_TIME` is set
+and not ``None``, it specifies the exact time of day for rollover and is used to calculate
+the initial rollover timing.
+
+
 Command-line options
 --------------------
 

scrapy/utils/log.py

@@ -3,6 +3,7 @@
 import logging
 import sys
 from logging.config import dictConfig
+from logging.handlers import TimedRotatingFileHandler
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
 
@@ -147,7 +148,19 @@ def _get_handler(settings: Settings) -> logging.Handler:
     if filename:
         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
         encoding = settings.get("LOG_ENCODING")
-        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
+        if settings.getbool("LOG_FILE_ROTATE", False):
+            handler = TimedRotatingFileHandler(
+                filename,
+                encoding=encoding,
+                when=settings.get("LOG_FILE_ROTATE_WHEN", "midnight"),
+                interval=settings.get("LOG_FILE_ROTATE_INTERVAL", 1),
+                backupCount=settings.get("LOG_FILE_ROTATE_BACKUP_COUNT", 0),
+                delay=settings.get("LOG_FILE_ROTATE_DELAY", False),
+                utc=settings.get("LOG_FILE_ROTATE_UTC", False),
+                atTime=settings.get("LOG_FILE_ROTATE_AT_TIME", None),
+            )
+        else:
+            handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
     elif settings.getbool("LOG_ENABLED"):
         handler = logging.StreamHandler()
     else:

tests/test_utils_log_rotate.py

@@ -0,0 +1,53 @@
+import datetime
+import logging
+import unittest
+
+from scrapy.utils.log import configure_logging
+
+
+class test_log_rotate(unittest.TestCase):
+    def setUp(self):
+        self.test_settings = {
+            "LOG_FILE": "sample.log",
+            "LOG_FILE_ROTATE": True,
+            "LOG_FILE_ROTATE_WHEN": "midnight",
+            "LOG_FILE_ROTATE_INTERVAL": 2,
+            "LOG_FILE_ROTATE_BACKUP_COUNT": 3,
+            "LOG_FILE_ROTATE_DELAY": True,
+            "LOG_FILE_ROTATE_UTC": True,
+            "LOG_FILE_ROTATE_AT_TIME": datetime.time(),
+        }
+        configure_logging(settings=self.test_settings)
+        self.handler = next(
+            (
+                handler
+                for handler in logging.root.handlers
+                if isinstance(handler, logging.handlers.TimedRotatingFileHandler)
+            ),
+            None,
+        )
+
+    def test_rotating_handler(self):
+        self.assertIsInstance(self.handler, logging.handlers.TimedRotatingFileHandler)
+
+    def test_attribute_when(self):
+        self.assertEqual(self.handler.when, "midnight".upper())
+
+    def test_attribute_interval(self):
+        self.assertEqual(self.handler.interval, 2 * 24 * 60 * 60)
+
+    def test_attribute_backupCount(self):
+        self.assertEqual(self.handler.backupCount, 3)
+
+    def test_attribute_delay(self):
+        self.assertEqual(self.handler.delay, True)
+
+    def test_attribute_utc(self):
+        self.assertEqual(self.handler.utc, True)
+
+    def test_attribute_atTime(self):
+        self.assertEqual(self.handler.atTime, datetime.time())
+
+
+if __name__ == "__main__":
+    unittest.main()