Add HEADERS_KEEP variable to settings and to HttpCompressionMiddlewar…

…e class (scrapy#1988)
VorBoto · Sep 14, 2019 · 5f068dc · 5f068dc
1 parent 534de73
commit 5f068dc
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
@@ -18,6 +18,10 @@
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
+
+    HEADERS_KEEP = setting.getbool('HEADERS_KEEP')
+
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
@@ -45,8 +49,9 @@ def process_response(self, request, response, spider):
                     # responsetypes guessing is reliable
                     kwargs['encoding'] = None
                 response = response.replace(**kwargs)
-                if not content_encoding:
-                    del response.headers['Content-Encoding']
+                if not HEADERS_KEEP:
+                    if not content_encoding:
+                        del response.headers['Content-Encoding']
 
         return response
 

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
@@ -170,6 +170,8 @@
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
+HEADERS_KEEP = False
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False