Added init to HttpCompMiddleware scrapy#1988

VorBoto · Sep 16, 2019 · 2b6fb1b · 2b6fb1b
1 parent 6a725ca
commit 2b6fb1b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
@@ -19,7 +19,10 @@ class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
 
-    HEADERS_KEEP = settings.getbool('HEADERS_KEEP')
+    def __int__(self, settings):
+        #setting to decide to keep or discard encoding header
+        #stored in default_settings.py
+        self.keep_encoding_header = setting.getbool('HTTPCOMPRESSION_HEADERS_KEEP')
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -48,7 +51,7 @@ def process_response(self, request, response, spider):
                     # responsetypes guessing is reliable
                     kwargs['encoding'] = None
                 response = response.replace(**kwargs)
-                if not HEADERS_KEEP:
+                if not self.keep_encoding_header:
                     if not content_encoding:
                         del response.headers['Content-Encoding']
 

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
@@ -170,8 +170,6 @@
 FTP_PASSWORD = 'guest'
 FTP_PASSIVE_MODE = True
 
-HEADERS_KEEP = False
-
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
@@ -185,6 +183,8 @@
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+HTTPCOMPRESSION_HEADERS_KEEP = False
+
 HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'