feat(scrapy#3628): add rotating log handler to support log rotation

docs/topics/logging.rst

@@ -146,6 +146,8 @@ These settings can be used to configure the logging:
 * :setting:`LOG_FILE_APPEND`
 * :setting:`LOG_ENABLED`
 * :setting:`LOG_ENCODING`
+* :setting:`LOG_MAX_BYTES`
+* :setting:`LOG_BACKUP_COUNT`
 * :setting:`LOG_LEVEL`
 * :setting:`LOG_FORMAT`
 * :setting:`LOG_DATEFORMAT`
@@ -158,7 +160,10 @@ redirected to a file named :setting:`LOG_FILE` with encoding
 :setting:`LOG_ENCODING`. If unset and :setting:`LOG_ENABLED` is ``True``, log
 messages will be displayed on the standard error. If :setting:`LOG_FILE` is set
 and :setting:`LOG_FILE_APPEND` is ``False``, the file will be overwritten
-(discarding the output from previous runs, if any). Lastly, if
+(discarding the output from previous runs, if any). When :setting:`LOG_FILE` is set,
+:setting:`LOG_MAX_BYTES` can be configured to set up log rotation to rotate logs when
+the log file reaches the specified size. :setting:`LOG_BACKUP_COUNT` can also be
+configured to manage the number of rotated log files that is kept on disk. Lastly, if
 :setting:`LOG_ENABLED` is ``False``, there won't be any visible log output.
 
 :setting:`LOG_LEVEL` determines the minimum level of severity to display, those

scrapy/utils/log.py

@@ -2,6 +2,7 @@
 import sys
 import warnings
 from logging.config import dictConfig
+from logging.handlers import RotatingFileHandler
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -130,7 +131,15 @@ def _get_handler(settings):
     if filename:
         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
         encoding = settings.get("LOG_ENCODING")
-        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
+        max_bytes = settings.get("LOG_MAX_BYTES", 0)
+        log_backup_count = settings.get("LOG_BACKUP_COUNT", 0)
+        handler = RotatingFileHandler(
+            filename,
+            mode=mode,
+            maxBytes=max_bytes,
+            backupCount=log_backup_count,
+            encoding=encoding,
+        )
     elif settings.getbool("LOG_ENABLED"):
         handler = logging.StreamHandler()
     else: