feat(ext/fetch): Replace reqwest with reqwest_middleware

Cargo.toml

@@ -146,6 +146,7 @@ prost-build = "0.11"
 rand = "=0.8.5"
 regex = "^1.7.0"
 reqwest = { version = "=0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
+reqwest-middleware = { version = "0.2.5" }
 ring = "^0.17.0"
 rusqlite = { version = "=0.29.0", features = ["unlock_notify", "bundled"] }
 rustls = "0.21.11"
@@ -168,6 +169,7 @@ spki = "0.7.2"
 tar = "=0.4.40"
 tempfile = "3.4.0"
 termcolor = "1.1.3"
+task-local-extensions = "0.1.4"
 thiserror = "1.0.40"
 tokio = { version = "1.36.0", features = ["full"] }
 tokio-metrics = { version = "0.3.0", features = ["rt"] }

cli/Cargo.toml

@@ -138,6 +138,7 @@ sha2.workspace = true
 shell-escape = "=0.1.5"
 spki = { version = "0.7", features = ["pem"] }
 tar.workspace = true
+task-local-extensions.workspace = true
 tempfile.workspace = true
 text-size = "=1.1.0"
 text_lines = "=0.6.0"

cli/http_util.rs

@@ -15,6 +15,7 @@ use deno_runtime::deno_fetch::create_http_client;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
 use deno_runtime::deno_fetch::reqwest::Response;
+use deno_runtime::deno_fetch::reqwest_middleware;
 use deno_runtime::deno_fetch::CreateHttpClientOptions;
 use deno_runtime::deno_tls::RootCertStoreProvider;
 use std::collections::HashMap;
@@ -222,7 +223,7 @@ impl CacheSemantics {
 pub struct HttpClient {
   options: CreateHttpClientOptions,
   root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
-  cell: once_cell::sync::OnceCell<reqwest::Client>,
+  cell: once_cell::sync::OnceCell<reqwest_middleware::ClientWithMiddleware>,
 }
 
 impl std::fmt::Debug for HttpClient {
@@ -249,7 +250,7 @@ impl HttpClient {
   }
 
   #[cfg(test)]
-  pub fn from_client(client: reqwest::Client) -> Self {
+  pub fn from_client(client: reqwest_middleware::ClientWithMiddleware) -> Self {
     let result = Self {
       options: Default::default(),
       root_cert_store_provider: Default::default(),
@@ -259,7 +260,9 @@ impl HttpClient {
     result
   }
 
-  pub(crate) fn client(&self) -> Result<&reqwest::Client, AnyError> {
+  pub(crate) fn client(
+    &self,
+  ) -> Result<&reqwest_middleware::ClientWithMiddleware, AnyError> {
     self.cell.get_or_try_init(|| {
       create_http_client(
         get_user_agent(),
@@ -278,7 +281,7 @@ impl HttpClient {
   pub fn get_no_redirect<U: reqwest::IntoUrl>(
     &self,
     url: U,
-  ) -> Result<reqwest::RequestBuilder, AnyError> {
+  ) -> Result<reqwest_middleware::RequestBuilder, AnyError> {
     Ok(self.client()?.get(url))
   }
 
@@ -390,6 +393,11 @@ pub async fn get_response_body_with_progress(
 mod test {
   use super::*;
 
+  use deno_runtime::deno_fetch::reqwest_middleware;
+  use std::sync::Arc;
+  use std::sync::Mutex;
+  use task_local_extensions;
+
   #[tokio::test]
   async fn test_http_client_download_redirect() {
     let _http_server_guard = test_util::http_server();
@@ -411,6 +419,54 @@ mod test {
     assert_eq!(err.to_string(), "Too many redirects.");
   }
 
+  #[tokio::test]
+  async fn test_http_client_middleware() {
+    // set up a middleware with a counter
+    struct CountingMiddleware {
+      count: Arc<Mutex<i32>>,
+    }
+
+    impl CountingMiddleware {
+      fn increment(&self) {
+        let mut count = self.count.lock().unwrap();
+        *count += 1;
+      }
+    }
+
+    #[async_trait::async_trait]
+    impl reqwest_middleware::Middleware for CountingMiddleware {
+      async fn handle(
+        &self,
+        req: reqwest::Request,
+        extensions: &mut task_local_extensions::Extensions,
+        next: reqwest_middleware::Next<'_>,
+      ) -> reqwest_middleware::Result<reqwest::Response> {
+        self.increment();
+        next.run(req, extensions).await
+      }
+    }
+
+    let _http_server_guard = test_util::http_server();
+
+    let count = Arc::new(Mutex::new(0));
+    let middleware = CountingMiddleware {
+      count: count.clone(),
+    };
+
+    deno_runtime::deno_fetch::add_middleware(Arc::new(middleware)).unwrap();
+    let client = HttpClient::new(None, None);
+
+    // make a request to the server and expect a 404 Not found
+    let err = client
+      .download_text("http://localhost:4545/")
+      .await
+      .err()
+      .unwrap();
+    assert_eq!(err.to_string(), "Not found.");
+    // check that the middleware has been called once
+    assert_eq!(*count.lock().unwrap(), 1);
+  }
+
   #[test]
   fn test_resolve_url_from_location_full_1() {
     let url = "http://deno.land".parse::<Url>().unwrap();

cli/tools/registry/api.rs

@@ -4,6 +4,7 @@ use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_runtime::deno_fetch::reqwest;
 use lsp_types::Url;
+use deno_runtime::deno_fetch::reqwest_middleware;
 use serde::de::DeserializeOwned;
 
 #[derive(serde::Deserialize)]
@@ -116,7 +117,7 @@ pub async fn parse_response<T: DeserializeOwned>(
 }
 
 pub async fn get_scope(
-  client: &reqwest::Client,
+  client: &reqwest_middleware::ClientWithMiddleware,
   registry_api_url: &str,
   scope: &str,
 ) -> Result<reqwest::Response, AnyError> {
@@ -134,7 +135,7 @@ pub fn get_package_api_url(
 }
 
 pub async fn get_package(
-  client: &reqwest::Client,
+  client: &reqwest_middleware::ClientWithMiddleware,
   registry_api_url: &str,
   scope: &str,
   package: &str,

cli/tools/registry/mod.rs

@@ -22,6 +22,7 @@ use deno_core::serde_json::json;
 use deno_core::serde_json::Value;
 use deno_core::unsync::JoinSet;
 use deno_runtime::deno_fetch::reqwest;
+use deno_runtime::deno_fetch::reqwest_middleware;
 use deno_terminal::colors;
 use import_map::ImportMap;
 use lsp_types::Url;
@@ -244,7 +245,7 @@ pub enum Permission<'s> {
 }
 
 async fn get_auth_headers(
-  client: &reqwest::Client,
+  client: &reqwest_middleware::ClientWithMiddleware,
   registry_url: String,
   packages: Vec<Rc<PreparedPublishPackage>>,
   auth_method: AuthMethod,
@@ -402,7 +403,7 @@ async fn get_auth_headers(
 /// Check if both `scope` and `package` already exist, if not return
 /// a URL to the management panel to create them.
 async fn check_if_scope_and_package_exist(
-  client: &reqwest::Client,
+  client: &reqwest_middleware::ClientWithMiddleware,
   registry_api_url: &str,
   registry_manage_url: &str,
   scope: &str,
@@ -434,7 +435,7 @@ async fn check_if_scope_and_package_exist(
 }
 
 async fn ensure_scopes_and_packages_exist(
-  client: &reqwest::Client,
+  client: &reqwest_middleware::ClientWithMiddleware,
   registry_api_url: String,
   registry_manage_url: String,
   packages: Vec<Rc<PreparedPublishPackage>>,

ext/fetch/Cargo.toml

@@ -20,8 +20,10 @@ deno_core.workspace = true
 deno_tls.workspace = true
 dyn-clone = "1"
 http_v02.workspace = true
+once_cell.workspace = true
 pin-project.workspace = true
 reqwest.workspace = true
+reqwest-middleware.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true