Follow vorboto/feature/httpcompression keeep response encoding

scrapy/downloadermiddlewares/httpcompression.py

@@ -9,6 +9,7 @@
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
+from scrapy.settings import Settings
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.gz import gunzip
 
@@ -33,18 +34,29 @@
     pass
 
 
+def _get_init_settings_kwargs(settings: Settings):
+    keep_encoding_header = (
+        settings.getbool("COMPRESSION_KEEP_ENCODING_HEADERS") if settings else False
+    )
+    return dict(keep_encoding_header=keep_encoding_header)
+
+
 class HttpCompressionMiddleware:
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    def __init__(self, stats: Optional[StatsCollector] = None):
+    def __init__(
+        self, stats: Optional[StatsCollector] = None, keep_encoding_header: bool = False
+    ):
         self.stats = stats
+        self.keep_encoding_header = keep_encoding_header
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         if not crawler.settings.getbool("COMPRESSION_ENABLED"):
             raise NotConfigured
-        return cls(stats=crawler.stats)
+        settings_kwargs = _get_init_settings_kwargs(crawler.settings)
+        return cls(stats=crawler.stats, **settings_kwargs)
 
     def process_request(
         self, request: Request, spider: Spider
@@ -79,10 +91,11 @@
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
                     kwargs["encoding"] = None
+                if self.keep_encoding_header:
+                    kwargs["flags"] = response.flags + ["decoded"]
                 response = response.replace(**kwargs)
-                if not content_encoding:
+                if not self.keep_encoding_header and not content_encoding:
                     del response.headers["Content-Encoding"]
-
         return response
 
     def _decode(self, body: bytes, encoding: bytes) -> bytes:

scrapy/settings/default_settings.py

@@ -39,6 +39,7 @@
 COMMANDS_MODULE = ""
 
 COMPRESSION_ENABLED = True
+COMPRESSION_KEEP_ENCODING_HEADERS = False
 
 CONCURRENT_ITEMS = 100
 

tests/test_downloadermiddleware_httpcompression.py

@@ -38,6 +38,17 @@
 
 
 class HttpCompressionTest(TestCase):
+    @staticmethod
+    def create_spider_mw(compression_enabled=True, compression_header=False):
+        settings = {
+            "COMPRESSION_ENABLED": compression_enabled,
+            "COMPRESSION_KEEP_ENCODING_HEADERS": compression_header,
+        }
+        crawler = get_crawler(Spider, settings)
+        spider = crawler._create_spider("foo")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        return spider, mw
+
     def setUp(self):
         self.crawler = get_crawler(Spider)
         self.spider = self.crawler._create_spider("scrapytest.org")
@@ -371,3 +382,31 @@ def test_process_response_head_request_no_decode_required(self):
         self.assertEqual(response.body, b"")
         self.assertStatsEqual("httpcompression/response_count", None)
         self.assertStatsEqual("httpcompression/response_bytes", None)
+
+    def test_process_response_gzip_keep_headers(self):
+        test_spider, test_mw = self.create_spider_mw(
+            compression_enabled=True, compression_header=True
+        )
+        response = self._getresponse("gzip")
+        request = response.request
+
+        self.assertEqual(response.headers["Content-Encoding"], b"gzip")
+        newresponse = test_mw.process_response(request, response, test_spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert "Content-Encoding" in newresponse.headers
+        assert "decoded" in newresponse.flags
+
+    def test_process_response_gzip_binary_octetstream_contenttype_kept(self):
+        test_spider, test_mw = self.create_spider_mw(
+            compression_enabled=True, compression_header=True
+        )
+        response = self._getresponse("x-gzip")
+        response.headers["Content-Type"] = "binary/octet-stream"
+        request = response.request
+
+        newresponse = test_mw.process_response(request, response, test_spider)
+        self.assertIsNot(newresponse, response)
+        self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
+        self.assertIn("Content-Encoding", newresponse.headers)
+        self.assertIn("decoded", newresponse.flags)