Improve type hints for copy() and replace() in Request and Response.

.github/workflows/checks.yml

@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs

scrapy/downloadermiddlewares/httpcompression.py

@@ -2,7 +2,7 @@
 
 import io
 import zlib
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
@@ -74,12 +74,12 @@ def process_response(
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs = dict(cls=respcls, body=decoded_body)
+                kwargs: Dict[str, Any] = dict(body=decoded_body)
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
                     kwargs["encoding"] = None
-                response = response.replace(**kwargs)
+                response = response.replace(cls=respcls, **kwargs)
                 if not content_encoding:
                     del response.headers["Content-Encoding"]
 

scrapy/downloadermiddlewares/redirect.py

@@ -27,6 +27,7 @@ def _build_redirect_request(
     redirect_request = source_request.replace(
         url=url,
         **kwargs,
+        cls=None,
         cookies=None,
     )
     if "Cookie" in redirect_request.headers:

scrapy/http/request/__init__.py

@@ -4,8 +4,11 @@
 
 See documentation in docs/topics/request-response.rst
 """
+from __future__ import annotations
+
 import inspect
 from typing import (
+    TYPE_CHECKING,
     Any,
     AnyStr,
     Callable,
@@ -19,7 +22,7 @@
     Type,
     TypeVar,
     Union,
-    cast,
+    overload,
 )
 
 from w3lib.url import safe_url_string
@@ -31,6 +34,11 @@
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
@@ -173,23 +181,36 @@ def encoding(self) -> str:
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 
-    def copy(self) -> "Request":
+    def copy(self) -> Self:
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> "Request":
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Request, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     @classmethod
     def from_curl(
-        cls: Type[RequestTypeVar],
+        cls,
         curl_command: str,
         ignore_unknown_options: bool = True,
         **kwargs: Any,
-    ) -> RequestTypeVar:
+    ) -> Self:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
         URL, the headers, the cookies and the body. It accepts the same
@@ -221,7 +242,7 @@ def from_curl(
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+    def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.

scrapy/http/request/json_request.py

@@ -5,12 +5,18 @@
 See documentation in docs/topics/request-response.rst
 """
 
+from __future__ import annotations
+
 import copy
 import json
 import warnings
-from typing import Any, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
+
+from scrapy.http.request import Request, RequestTypeVar
 
-from scrapy.http.request import Request
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class JsonRequest(Request):
@@ -44,7 +50,19 @@ def __init__(
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args: Any, **kwargs: Any) -> Request:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         body_passed = kwargs.get("body", None) is not None
         data = kwargs.pop("data", None)
         data_passed = data is not None
@@ -54,7 +72,7 @@ def replace(self, *args: Any, **kwargs: Any) -> Request:
         elif not body_passed and data_passed:
             kwargs["body"] = self._dumps(data)
 
-        return super().replace(*args, **kwargs)
+        return super().replace(*args, cls=cls, **kwargs)
 
     def _dumps(self, data: dict) -> str:
         """Convert to JSON"""

scrapy/http/response/__init__.py

@@ -19,8 +19,10 @@
     Mapping,
     Optional,
     Tuple,
+    Type,
+    TypeVar,
     Union,
-    cast,
+    overload,
 )
 from urllib.parse import urljoin
 
@@ -33,9 +35,15 @@
 from scrapy.utils.trackref import object_ref
 
 if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
     from scrapy.selector import SelectorList
 
 
+ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
+
+
 class Response(object_ref):
     """An object that represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -132,16 +140,29 @@ def _set_body(self, body: Optional[bytes]) -> None:
     def __repr__(self) -> str:
         return f"<{self.status} {self.url}>"
 
-    def copy(self) -> Response:
+    def copy(self) -> Self:
         """Return a copy of this Response"""
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> Response:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
+    ) -> ResponseTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
+    ) -> Response:
         """Create a new Response with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Response, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     def urljoin(self, url: str) -> str:
         """Join this Response's url with a possible relative url to form an

tests_typing/test_http_request.mypy-testing

@@ -0,0 +1,66 @@
+import pytest
+
+from scrapy import Request
+from scrapy.http import JsonRequest
+
+
+class MyRequest(Request):
+    pass
+
+
+class MyRequest2(Request):
+    pass
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
+    Request("data:,", headers=None)
+    Request("data:,", headers={})
+    Request("data:,", headers=[])
+    Request("data:,", headers={"foo": "bar"})
+    Request("data:,", headers={b"foo": "bar"})
+    Request("data:,", headers={"foo": b"bar"})
+    Request("data:,", headers=[("foo", "bar")])
+    Request("data:,", headers=[(b"foo", "bar")])
+    Request("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: __main__.MyRequest
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: __main__.MyRequest
+    req_copy2 = req.replace(body=b"a", cls=MyRequest2)
+    reveal_type(req_copy2)  # R: __main__.MyRequest2
+
+
+@pytest.mark.mypy_testing
+def mypy_test_jsonrequest_copy_replace():
+    req = JsonRequest("data:,")
+    reveal_type(req)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy_my = req.replace(body=b"a", cls=MyRequest)
+    reveal_type(req_copy_my)  # R: __main__.MyRequest

tests_typing/test_http_response.mypy-testing

@@ -0,0 +1,45 @@
+import pytest
+
+from scrapy.http import HtmlResponse, Response, TextResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
+    Response("data:,", headers=None)
+    Response("data:,", headers={})
+    Response("data:,", headers=[])
+    Response("data:,", headers={"foo": "bar"})
+    Response("data:,", headers={b"foo": "bar"})
+    Response("data:,", headers={"foo": b"bar"})
+    Response("data:,", headers=[("foo", "bar")])
+    Response("data:,", headers=[(b"foo", "bar")])
+    Response("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
+    reveal_type(resp_copy2)  # R: scrapy.http.response.text.TextResponse

tox.ini

@@ -46,6 +46,14 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+deps =
+    -rtests/requirements.txt
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.1
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =