Fix HTTP3 stress (#69182)

* Build msquic locally

* Use IPv4 addresses only when testing HTTP3

* Code review changes

* fixup! Use IPv4 addresses only when testing HTTP3

* Make listening on IPv6Any accept IPv4 connections as well

* Minor changes

* Don't continue if runtime image build fails

* Switch to 1es-ubuntu-1804-open image

* Code review feedback

eng/pipelines/libraries/stress/http.yml

@@ -31,21 +31,21 @@ jobs:
     DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
   pool:
     name: NetCore1ESPool-Public
-    demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+    demands: ImageOverride -equals 1es-ubuntu-1804-open
 
   steps:
   - checkout: self
     clean: true
     fetchDepth: 5
 
   - bash: |
-      $(dockerfilesFolder)/build-docker-sdk.sh -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION)
+      $(dockerfilesFolder)/build-docker-sdk.sh -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION) && \
       echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
     name: buildRuntime
     displayName: Build CLR and Libraries
 
   - bash: |
-      $(httpStressProject)/run-docker-compose.sh -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage)
+      $(httpStressProject)/run-docker-compose.sh -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage) && \
       echo "##vso[task.setvariable variable=succeeded;isOutput=true]true"
     name: buildStress
     displayName: Build HttpStress

src/libraries/Common/src/System/Net/SocketAddress.cs

@@ -133,6 +133,7 @@ internal SocketAddress(AddressFamily addressFamily, ReadOnlySpan<byte> buffer)
         {
             Buffer = buffer.ToArray();
             InternalSize = Buffer.Length;
+            SocketAddressPal.SetAddressFamily(Buffer, addressFamily);
         }
 
         internal IPAddress GetIPAddress()

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile

@@ -1,21 +1,32 @@
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 FROM $SDK_BASE_IMAGE
 
-WORKDIR /app
-COPY . .
-
-# Pulling the msquic Debian package from packages.microsoft.com
-RUN apt-get update -y
-RUN apt-get install -y gnupg2 software-properties-common
-RUN curl -sSL https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
-RUN apt-add-repository https://packages.microsoft.com/debian/11/prod
-RUN apt-get update -y
-RUN apt-get install -y libmsquic
-RUN apt-get upgrade -y
+# Build latest msquic locally
+WORKDIR /msquic
+RUN apt-get update -y && \
+    apt-get upgrade -y && \
+    apt-get install -y cmake clang ruby-dev gem lttng-tools libssl-dev && \
+    gem install fpm
+RUN git clone --recursive https://github.com/dotnet/msquic
+RUN cd msquic/src/msquic && \
+    mkdir build && \
+    cmake -B build -DCMAKE_BUILD_TYPE=Release -DQUIC_ENABLE_LOGGING=false -DQUIC_USE_SYSTEM_LIBCRYPTO=true -DQUIC_BUILD_TOOLS=off -DQUIC_BUILD_TEST=off -DQUIC_BUILD_PERF=off && \
+    cd build && \
+    cmake --build . --config Release
+RUN cd msquic/src/msquic/build/bin/Release && \
+    rm libmsquic.so && \
+    fpm -f -s dir -t deb -n libmsquic -v $( find -type f | cut -d "." -f 4- ) \
+    --license MIT --url https://github.com/microsoft/msquic --log error \
+    $( ls ./* | cut -d "/" -f 2 | sed -r "s/(.*)/\1=\/usr\/lib\/\1/g" ) && \
+    dpkg -i libmsquic_*.deb
 
 ARG VERSION=7.0
 ARG CONFIGURATION=Release
 
+# Build the stress server
+WORKDIR /app
+COPY . .
+
 RUN dotnet build -c $CONFIGURATION \
     -p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \
     -p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \

src/libraries/System.Net.Http/tests/StressTests/HttpStress/StressServer.cs

@@ -118,9 +118,9 @@ void ConfigureListenOptions(ListenOptions listenOptions)
                         else
                         {
                             listenOptions.Protocols =
-                                configuration.HttpVersion ==  HttpVersion.Version20 ?
+                                configuration.HttpVersion == HttpVersion.Version20 ?
                                 HttpProtocols.Http2 :
-                                HttpProtocols.Http1 ;
+                                HttpProtocols.Http1;
                         }
                     }
                 });
@@ -139,7 +139,8 @@ void ConfigureListenOptions(ListenOptions listenOptions)
                     try
                     {
                         File.Delete(filename);
-                    } catch {}
+                    }
+                    catch { }
                 }
 
                 loggerConfiguration = loggerConfiguration

src/libraries/System.Net.Quic/src/System/Net/Quic/Implementations/MsQuic/Internal/MsQuicAddressHelpers.cs

@@ -1,6 +1,11 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Diagnostics;
+using System.Net.Sockets;
+using System.Runtime.InteropServices;
+using Microsoft.Quic;
+
 namespace System.Net.Quic.Implementations.MsQuic.Internal
 {
     internal static class MsQuicAddressHelpers
@@ -11,5 +16,18 @@ internal static unsafe IPEndPoint INetToIPEndPoint(IntPtr pInetAddress)
             Span<byte> addressBytes = new Span<byte>((byte*)pInetAddress, Internals.SocketAddress.IPv6AddressSize);
             return new Internals.SocketAddress(SocketAddressPal.GetAddressFamily(addressBytes), addressBytes).GetIPEndPoint();
         }
+
+        internal static unsafe QuicAddr ToQuicAddr(this IPEndPoint iPEndPoint)
+        {
+            // TODO: is the layout same for SocketAddress.Buffer and QuicAddr on all platforms?
+            QuicAddr result = default;
+            Span<byte> rawAddress = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref result, 1));
+
+            Internals.SocketAddress address = IPEndPointExtensions.Serialize(iPEndPoint);
+            Debug.Assert(address.Size <= rawAddress.Length);
+
+            address.Buffer.AsSpan(0, address.Size).CopyTo(rawAddress);
+            return result;
+        }
     }
 }

src/libraries/System.Net.Quic/src/System/Net/Quic/Implementations/MsQuic/Internal/MsQuicParameterHelpers.cs

@@ -4,13 +4,14 @@
 using System.Diagnostics;
 using System.Runtime.InteropServices;
 using System.Net.Sockets;
+using Microsoft.Quic;
 using static Microsoft.Quic.MsQuic;
 
 namespace System.Net.Quic.Implementations.MsQuic.Internal
 {
     internal static class MsQuicParameterHelpers
     {
-        internal static unsafe IPEndPoint GetIPEndPointParam(MsQuicApi api, MsQuicSafeHandle nativeObject, uint param)
+        internal static unsafe IPEndPoint GetIPEndPointParam(MsQuicApi api, MsQuicSafeHandle nativeObject, uint param, AddressFamily? addressFamilyOverride = null)
         {
             // MsQuic always uses storage size as if IPv6 was used
             uint valueLen = (uint)Internals.SocketAddress.IPv6AddressSize;
@@ -27,7 +28,7 @@ internal static unsafe IPEndPoint GetIPEndPointParam(MsQuicApi api, MsQuicSafeHa
 
             address = address.Slice(0, (int)valueLen);
 
-            return new Internals.SocketAddress(SocketAddressPal.GetAddressFamily(address), address).GetIPEndPoint();
+            return new Internals.SocketAddress(addressFamilyOverride ?? SocketAddressPal.GetAddressFamily(address), address).GetIPEndPoint();
         }
 
         internal static unsafe void SetIPEndPointParam(MsQuicApi api, MsQuicSafeHandle nativeObject, uint param, IPEndPoint value)

src/libraries/System.Net.Quic/src/System/Net/Quic/Implementations/MsQuic/MsQuicListener.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Buffers;
+using System.Buffers.Binary;
 using System.Collections.Generic;
 using System.Collections.Concurrent;
 using System.Diagnostics;
@@ -12,7 +13,6 @@
 using System.Threading;
 using System.Threading.Channels;
 using System.Threading.Tasks;
-using System.Net.Sockets;
 using Microsoft.Quic;
 using System.Runtime.CompilerServices;
 using static Microsoft.Quic.MsQuic;
@@ -179,26 +179,27 @@ private unsafe IPEndPoint Start(QuicListenerOptions options)
             List<SslApplicationProtocol> applicationProtocols = options.ServerAuthenticationOptions!.ApplicationProtocols!;
             IPEndPoint listenEndPoint = options.ListenEndPoint!;
 
-            Internals.SocketAddress address = IPEndPointExtensions.Serialize(listenEndPoint);
-
             Debug.Assert(_stateHandle.IsAllocated);
             try
             {
                 Debug.Assert(!Monitor.IsEntered(_state), "!Monitor.IsEntered(_state)");
                 using var msquicBuffers = new MsQuicBuffers();
                 msquicBuffers.Initialize(applicationProtocols, applicationProtocol => applicationProtocol.Protocol);
-                // TODO: is the layout same for SocketAddress.Buffer and QuicAddr?
-                // TODO: maybe add simple extensions/helpers:
-                //       - QuicAddr ToQuicAddr(this IPEndPoint ipEndPoint)
-                //       - IPEndPoint ToIPEndPoint(this ref QuicAddr quicAddress)
-                fixed (byte* paddress = address.Buffer)
+
+                QuicAddr address = listenEndPoint.ToQuicAddr();
+
+                if (listenEndPoint.Address == IPAddress.IPv6Any)
                 {
-                    ThrowIfFailure(MsQuicApi.Api.ApiTable->ListenerStart(
-                        _state.Handle.QuicHandle,
-                        msquicBuffers.Buffers,
-                        (uint)applicationProtocols.Count,
-                        (QuicAddr*)paddress), "ListenerStart failed");
+                    // For IPv6Any, MsQuic would listen only for IPv6 connections. To mimic the behavior of TCP sockets,
+                    // we leave the address family unspecified and let MsQuic handle connections from all IP addresses.
+                    address.Family = QUIC_ADDRESS_FAMILY_UNSPEC;
                 }
+
+                ThrowIfFailure(MsQuicApi.Api.ApiTable->ListenerStart(
+                    _state.Handle.QuicHandle,
+                    msquicBuffers.Buffers,
+                    (uint)applicationProtocols.Count,
+                    &address), "ListenerStart failed");
             }
             catch
             {
@@ -207,7 +208,8 @@ private unsafe IPEndPoint Start(QuicListenerOptions options)
             }
 
             Debug.Assert(!Monitor.IsEntered(_state), "!Monitor.IsEntered(_state)");
-            return MsQuicParameterHelpers.GetIPEndPointParam(MsQuicApi.Api, _state.Handle, QUIC_PARAM_LISTENER_LOCAL_ADDRESS);
+            // override the address family to the original value in case we had to use UNSPEC
+            return MsQuicParameterHelpers.GetIPEndPointParam(MsQuicApi.Api, _state.Handle, QUIC_PARAM_LISTENER_LOCAL_ADDRESS, listenEndPoint.AddressFamily);
         }
 
         private unsafe Task StopAsync()

src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs

@@ -41,6 +41,21 @@ public async Task Listener_Backlog_Success_IPv6()
                 await clientStreamTask;
             }).WaitAsync(TimeSpan.FromSeconds(6));
         }
+
+        [ConditionalFact(nameof(IsMsQuicProvider))]
+        public async Task Listener_IPv6Any_Accepts_IPv4()
+        {
+            await Task.Run(async () =>
+            {
+                using QuicListener listener = CreateQuicListener(new IPEndPoint(IPAddress.IPv6Any, 0));
+
+                using QuicConnection clientConnection = CreateQuicConnection(new IPEndPoint(IPAddress.Loopback, listener.ListenEndPoint.Port));
+                var clientStreamTask = clientConnection.ConnectAsync();
+
+                using QuicConnection serverConnection = await listener.AcceptConnectionAsync();
+                await clientStreamTask;
+            }).WaitAsync(TimeSpan.FromSeconds(6));
+        }
     }
 
     [ConditionalClass(typeof(QuicTestBase<MockProviderFactory>), nameof(QuicTestBase<MockProviderFactory>.IsSupported))]