feat: upgrade Reqwest to 0.12 [WIP]

Cargo.toml

@@ -113,12 +113,12 @@ fs3 = "0.5.0"
 futures = "0.3.21"
 glob = "0.3.1"
 h2 = "0.4.4"
-http = "1.0"
+http = "1.1"
 http-body-util = "0.1"
 http_v02 = { package = "http", version = "0.2.9" }
 httparse = "1.8.0"
 hyper = { version = "=1.1.0", features = ["full"] }
-hyper-util = { version = "=0.1.2", features = ["tokio", "server", "server-auto"] }
+hyper-util = { version = "=0.1.3", default-features = false, features = ["tokio", "server", "server-auto"] }
 hyper_v014 = { package = "hyper", version = "0.14.26", features = ["runtime", "http1"] }
 indexmap = { version = "2", features = ["serde"] }
 jsonc-parser = { version = "=0.23.0", features = ["serde"] }
@@ -145,12 +145,12 @@ prost = "0.11"
 prost-build = "0.11"
 rand = "=0.8.5"
 regex = "^1.7.0"
-reqwest = { version = "=0.11.20", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
+reqwest = { version = "=0.12.4", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json", "http2"] } # pinned because of https://github.com/seanmonstar/reqwest/pull/1955
 ring = "^0.17.0"
 rusqlite = { version = "=0.29.0", features = ["unlock_notify", "bundled"] }
-rustls = "0.21.11"
+rustls = "0.22.4"
 rustls-pemfile = "1.0.0"
-rustls-tokio-stream = "=0.2.17"
+rustls-tokio-stream = "0.2.20"
 rustls-webpki = "0.101.4"
 rustyline = "=13.0.0"
 saffron = "=0.1.0"
@@ -176,7 +176,7 @@ tower-lsp = { version = "=0.20.0", features = ["proposed"] }
 # Upgrading past 2.4.1 may cause WPT failures
 url = { version = "< 2.5.0", features = ["serde", "expose_internals"] }
 uuid = { version = "1.3.0", features = ["v4"] }
-webpki-roots = "0.25.2"
+webpki-roots = "0.26.1"
 zeromq = { version = "=0.3.4", default-features = false, features = ["tcp-transport", "tokio-runtime"] }
 zstd = "=0.12.4"
 

cli/args/mod.rs

@@ -14,7 +14,10 @@ use deno_ast::SourceMapOption;
 use deno_core::resolve_url_or_path;
 use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
 use deno_npm::NpmSystemInfo;
+use deno_runtime::deno_tls::create_default_root_cert_store;
+use deno_runtime::deno_tls::create_platform_cert_store;
 use deno_runtime::deno_tls::RootCertStoreProvider;
+use deno_runtime::permissions::PermissionsOptions;
 use deno_semver::npm::NpmPackageReqReference;
 use indexmap::IndexMap;
 
@@ -43,13 +46,10 @@ use deno_core::parking_lot::Mutex;
 use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_runtime::deno_node::PackageJson;
-use deno_runtime::deno_tls::deno_native_certs::load_native_certs;
-use deno_runtime::deno_tls::rustls;
+
 use deno_runtime::deno_tls::rustls::RootCertStore;
 use deno_runtime::deno_tls::rustls_pemfile;
-use deno_runtime::deno_tls::webpki_roots;
 use deno_runtime::inspector_server::InspectorServer;
-use deno_runtime::permissions::PermissionsOptions;
 use deno_terminal::colors;
 use dotenvy::from_filename;
 use once_cell::sync::Lazy;
@@ -601,7 +601,6 @@ pub fn get_root_cert_store(
   maybe_ca_stores: Option<Vec<String>>,
   maybe_ca_data: Option<CaData>,
 ) -> Result<RootCertStore, RootCertStoreLoadError> {
-  let mut root_cert_store = RootCertStore::empty();
   let ca_stores: Vec<String> = maybe_ca_stores
     .or_else(|| {
       let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?;
@@ -615,26 +614,18 @@ pub fn get_root_cert_store(
     })
     .unwrap_or_else(|| vec!["mozilla".to_string()]);
 
+  let mut root_cert_store = RootCertStore::empty();
   for store in ca_stores.iter() {
     match store.as_str() {
       "mozilla" => {
-        root_cert_store.add_trust_anchors(
-          webpki_roots::TLS_SERVER_ROOTS.iter().map(|ta| {
-            rustls::OwnedTrustAnchor::from_subject_spki_name_constraints(
-              ta.subject,
-              ta.spki,
-              ta.name_constraints,
-            )
-          }),
-        );
+        root_cert_store
+          .roots
+          .extend(create_default_root_cert_store().roots);
       }
       "system" => {
-        let roots = load_native_certs().expect("could not load platform certs");
-        for root in roots {
-          root_cert_store
-            .add(&rustls::Certificate(root.0))
-            .expect("Failed to add platform cert to root cert store");
-        }
+        root_cert_store
+          .roots
+          .extend(create_platform_cert_store().roots);
       }
       _ => {
         return Err(RootCertStoreLoadError::UnknownStore(store.clone()));
@@ -665,8 +656,9 @@ pub fn get_root_cert_store(
     };
 
     match result {
-      Ok(certs) => {
-        root_cert_store.add_parsable_certificates(&certs);
+      Ok(_certs) => {
+        unreachable!()
+        // root_cert_store.add(&certs);
       }
       Err(e) => {
         return Err(RootCertStoreLoadError::FailedAddPemFile(e.to_string()));

ext/fetch/Cargo.toml

@@ -19,7 +19,7 @@ data-url.workspace = true
 deno_core.workspace = true
 deno_tls.workspace = true
 dyn-clone = "1"
-http_v02.workspace = true
+http.workspace = true
 pin-project.workspace = true
 reqwest.workspace = true
 serde.workspace = true

ext/fetch/fs_fetch_handler.rs

@@ -31,7 +31,7 @@ impl FetchHandler for FsFetchHandler {
       let file = tokio::fs::File::open(path).map_err(|_| ()).await?;
       let stream = ReaderStream::new(file);
       let body = reqwest::Body::wrap_stream(stream);
-      let response = http_v02::Response::builder()
+      let response = http::Response::builder()
         .status(StatusCode::OK)
         .body(body)
         .map_err(|_| ())?

ext/fetch/lib.rs

@@ -46,8 +46,8 @@ use deno_tls::RootCertStoreProvider;
 use data_url::DataUrl;
 use deno_tls::TlsKey;
 use deno_tls::TlsKeys;
-use http_v02::header::CONTENT_LENGTH;
-use http_v02::Uri;
+use http::header::CONTENT_LENGTH;
+use http::Uri;
 use reqwest::header::HeaderMap;
 use reqwest::header::HeaderName;
 use reqwest::header::HeaderValue;
@@ -418,12 +418,9 @@ where
         .decode_to_vec()
         .map_err(|e| type_error(format!("{e:?}")))?;
 
-      let response = http_v02::Response::builder()
-        .status(http_v02::StatusCode::OK)
-        .header(
-          http_v02::header::CONTENT_TYPE,
-          data_url.mime_type().to_string(),
-        )
+      let response = http::Response::builder()
+        .status(http::StatusCode::OK)
+        .header(http::header::CONTENT_TYPE, data_url.mime_type().to_string())
         .body(reqwest::Body::from(body))?;
 
       let fut = async move { Ok(Ok(Response::from(response))) };

ext/kv/Cargo.toml

@@ -39,5 +39,7 @@ tokio.workspace = true
 url.workspace = true
 uuid = { workspace = true, features = ["serde"] }
 
+reqwest_11 = { package = "reqwest", version = "=0.11.27", default-features = false, features = ["rustls-tls", "stream", "gzip", "brotli", "socks", "json"] }
+
 [build-dependencies]
 prost-build.workspace = true

ext/kv/remote.rs

@@ -11,14 +11,17 @@ use async_trait::async_trait;
 use deno_core::error::type_error;
 use deno_core::error::AnyError;
 use deno_core::OpState;
-use deno_fetch::create_http_client;
 use deno_fetch::CreateHttpClientOptions;
 use deno_tls::rustls::RootCertStore;
 use deno_tls::Proxy;
 use deno_tls::RootCertStoreProvider;
 use deno_tls::TlsKey;
 use denokv_remote::MetadataEndpoint;
 use denokv_remote::Remote;
+use reqwest_11::header::HeaderMap;
+use reqwest_11::header::USER_AGENT;
+use reqwest_11::redirect::Policy;
+use reqwest_11::Client;
 use url::Url;
 
 #[derive(Clone)]
@@ -86,6 +89,67 @@ impl<P: RemoteDbHandlerPermissions + 'static> denokv_remote::RemotePermissions
   }
 }
 
+/// Create new instance of async reqwest::Client. This client supports
+/// proxies and doesn't follow redirects.
+pub fn create_http_client(
+  user_agent: &str,
+  options: CreateHttpClientOptions,
+) -> Result<Client, AnyError> {
+  let mut tls_config = deno_tls::create_client_config(
+    options.root_cert_store,
+    options.ca_certs,
+    options.unsafely_ignore_certificate_errors,
+    options.client_cert_chain_and_key,
+    deno_tls::SocketUse::Http,
+  )?;
+
+  let mut alpn_protocols = vec![];
+  if options.http2 {
+    alpn_protocols.push("h2".into());
+  }
+  if options.http1 {
+    alpn_protocols.push("http/1.1".into());
+  }
+  tls_config.alpn_protocols = alpn_protocols;
+
+  let mut headers = HeaderMap::new();
+  headers.insert(USER_AGENT, user_agent.parse().unwrap());
+  let mut builder = Client::builder()
+    .redirect(Policy::none())
+    .default_headers(headers)
+    .use_preconfigured_tls(tls_config);
+
+  if let Some(proxy) = options.proxy {
+    let mut reqwest_proxy = reqwest_11::Proxy::all(&proxy.url)?;
+    if let Some(basic_auth) = &proxy.basic_auth {
+      reqwest_proxy =
+        reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password);
+    }
+    builder = builder.proxy(reqwest_proxy);
+  }
+
+  if let Some(pool_max_idle_per_host) = options.pool_max_idle_per_host {
+    builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+  }
+
+  if let Some(pool_idle_timeout) = options.pool_idle_timeout {
+    builder = builder.pool_idle_timeout(
+      pool_idle_timeout.map(std::time::Duration::from_millis),
+    );
+  }
+
+  match (options.http1, options.http2) {
+    (true, false) => builder = builder.http1_only(),
+    (false, true) => builder = builder.http2_prior_knowledge(),
+    (true, true) => {}
+    (false, false) => {
+      return Err(type_error("Either `http1` or `http2` needs to be true"))
+    }
+  }
+
+  builder.build().map_err(|e| e.into())
+}
+
 #[async_trait(?Send)]
 impl<P: RemoteDbHandlerPermissions + 'static> DatabaseHandler
   for RemoteDbHandler<P>

ext/net/ops_tls.rs

@@ -29,10 +29,11 @@ use deno_core::ResourceId;
 use deno_tls::create_client_config;
 use deno_tls::load_certs;
 use deno_tls::load_private_keys;
-use deno_tls::rustls::Certificate;
-use deno_tls::rustls::PrivateKey;
+use deno_tls::rustls::pki_types::ServerName;
+use deno_tls::rustls::ClientConnection;
 use deno_tls::rustls::ServerConfig;
-use deno_tls::rustls::ServerName;
+use deno_tls::Certificate;
+use deno_tls::PrivateKey;
 use deno_tls::SocketUse;
 use deno_tls::TlsKey;
 use deno_tls::TlsKeys;
@@ -241,12 +242,13 @@ where
   let hostname = match &*args.hostname {
     "" => "localhost",
     n => n,
-  };
+  }
+  .to_owned();
 
   {
     let mut s = state.borrow_mut();
     let permissions = s.borrow_mut::<NP>();
-    permissions.check_net(&(hostname, Some(0)), "Deno.startTls()")?;
+    permissions.check_net(&(&hostname, Some(0)), "Deno.startTls()")?;
   }
 
   let ca_certs = args
@@ -255,8 +257,8 @@ where
     .map(|s| s.into_bytes())
     .collect::<Vec<_>>();
 
-  let hostname_dns =
-    ServerName::try_from(hostname).map_err(|_| invalid_hostname(hostname))?;
+  let hostname_dns = ServerName::try_from(hostname.clone())
+    .map_err(|_| invalid_hostname(&hostname))?;
 
   let unsafely_ignore_certificate_errors = state
     .borrow()
@@ -299,8 +301,7 @@ where
   let tls_config = Arc::new(tls_config);
   let tls_stream = TlsStream::new_client_side(
     tcp_stream,
-    tls_config,
-    hostname_dns,
+    ClientConnection::new(tls_config, hostname_dns).unwrap(),
     TLS_BUFFER_SIZE,
   );
 
@@ -357,7 +358,7 @@ where
     .borrow()
     .borrow::<DefaultTlsOptions>()
     .root_cert_store()?;
-  let hostname_dns = ServerName::try_from(&*addr.hostname)
+  let hostname_dns = ServerName::try_from(addr.hostname.clone())
     .map_err(|_| invalid_hostname(&addr.hostname))?;
   let connect_addr = resolve_addr(&addr.hostname, addr.port)
     .await?
@@ -388,8 +389,7 @@ where
 
   let tls_stream = TlsStream::new_client_side(
     tcp_stream,
-    tls_config,
-    hostname_dns,
+    ClientConnection::new(tls_config, hostname_dns).unwrap(),
     TLS_BUFFER_SIZE,
   );
 
@@ -444,14 +444,12 @@ where
       .check_net(&(&addr.hostname, Some(addr.port)), "Deno.listenTls()")?;
   }
 
-  let tls_config = ServerConfig::builder()
-    .with_safe_defaults()
-    .with_no_client_auth();
+  let tls_config = ServerConfig::builder().with_no_client_auth();
 
   let mut tls_config = match keys {
     TlsKeys::Null => Err(anyhow!("Deno.listenTls requires a key")),
     TlsKeys::Static(TlsKey(cert, key)) => tls_config
-      .with_single_cert(cert.clone(), key.clone())
+      .with_single_cert(cert.clone(), key.clone_key())
       .map_err(|e| anyhow!(e)),
   }
   .map_err(|e| {

ext/node/Cargo.toml

@@ -34,9 +34,9 @@ ecb.workspace = true
 elliptic-curve.workspace = true
 errno = "0.2.8"
 faster-hex.workspace = true
-h2 = { version = "0.3.26", features = ["unstable"] }
+h2 = { version = "0.4.4", features = ["unstable"] }
 hkdf.workspace = true
-http_v02.workspace = true
+http.workspace = true
 idna = "0.3.0"
 indexmap.workspace = true
 k256 = "0.13.1"

ext/node/ops/http2.rs

@@ -25,11 +25,11 @@ use deno_net::raw::take_network_stream_resource;
 use deno_net::raw::NetworkStream;
 use h2;
 use h2::RecvStream;
-use http_v02;
-use http_v02::request::Parts;
-use http_v02::HeaderMap;
-use http_v02::Response;
-use http_v02::StatusCode;
+use http;
+use http::request::Parts;
+use http::HeaderMap;
+use http::Response;
+use http::StatusCode;
 use reqwest::header::HeaderName;
 use reqwest::header::HeaderValue;
 use url::Url;
@@ -310,7 +310,7 @@ pub async fn op_http2_client_request(
 
   let url = url.join(&pseudo_path)?;
 
-  let mut req = http_v02::Request::builder()
+  let mut req = http::Request::builder()
     .uri(url.as_str())
     .method(pseudo_method.as_str());
 
@@ -398,7 +398,7 @@ pub async fn op_http2_client_send_trailers(
     .get::<Http2ClientStream>(stream_rid)?;
   let mut stream = RcRef::map(&resource, |r| &r.stream).borrow_mut().await;
 
-  let mut trailers_map = http_v02::HeaderMap::new();
+  let mut trailers_map = http::HeaderMap::new();
   for (name, value) in trailers {
     trailers_map.insert(
       HeaderName::from_bytes(&name).unwrap(),