[release/8.0] Change how ANCM recycles app (#55288)

* Change how ANCM recycles app

* app init and single join

* don't recycle again

* fix iisexpress

* fallback

* testing

* cleanup

* env var

* fb

* fb

* compile

* Apply suggestions from code review

* Apply suggestions from code review

---------

Co-authored-by: Brennan <brecon@microsoft.com>

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp

@@ -22,7 +22,8 @@ const PCWSTR HandlerResolver::s_pwzAspnetcoreOutOfProcessRequestHandlerName = L"
 HandlerResolver::HandlerResolver(HMODULE hModule, const IHttpServer &pServer)
     : m_hModule(hModule),
       m_pServer(pServer),
-      m_loadedApplicationHostingModel(HOSTING_UNKNOWN)
+      m_loadedApplicationHostingModel(HOSTING_UNKNOWN),
+      m_shutdownDelay()
 {
     m_disallowRotationOnConfigChange = false;
     InitializeSRWLock(&m_requestHandlerLoadLock);
@@ -171,6 +172,7 @@ HandlerResolver::GetApplicationFactory(const IHttpApplication& pApplication, con
     m_loadedApplicationHostingModel = options.QueryHostingModel();
     m_loadedApplicationId = pApplication.GetApplicationId();
     m_disallowRotationOnConfigChange = options.QueryDisallowRotationOnConfigChange();
+    m_shutdownDelay = options.QueryShutdownDelay();
 
     RETURN_IF_FAILED(LoadRequestHandlerAssembly(pApplication, shadowCopyPath, options, pApplicationFactory, errorContext));
 
@@ -197,6 +199,11 @@ bool HandlerResolver::GetDisallowRotationOnConfigChange()
     return m_disallowRotationOnConfigChange;
 }
 
+std::chrono::milliseconds HandlerResolver::GetShutdownDelay() const
+{
+    return m_shutdownDelay;
+}
+
 HRESULT
 HandlerResolver::FindNativeAssemblyFromGlobalLocation(
     const ShimOptions& pConfiguration,

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h

@@ -19,6 +19,7 @@ class HandlerResolver
     void ResetHostingModel();
     APP_HOSTING_MODEL GetHostingModel();
     bool GetDisallowRotationOnConfigChange();
+    std::chrono::milliseconds GetShutdownDelay() const;
 
 private:
     HRESULT LoadRequestHandlerAssembly(const IHttpApplication &pApplication, const std::filesystem::path& shadowCopyPath, const ShimOptions& pConfiguration, std::unique_ptr<ApplicationFactory>& pApplicationFactory, ErrorContext& errorContext);
@@ -40,6 +41,7 @@ class HandlerResolver
     APP_HOSTING_MODEL m_loadedApplicationHostingModel;
     HostFxr m_hHostFxrDll;
     bool m_disallowRotationOnConfigChange;
+    std::chrono::milliseconds m_shutdownDelay;
 
     static const PCWSTR          s_pwzAspnetcoreInProcessRequestHandlerName;
     static const PCWSTR          s_pwzAspnetcoreOutOfProcessRequestHandlerName;

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp

@@ -12,6 +12,8 @@
 #define CS_ASPNETCORE_SHADOW_COPY_DIRECTORY              L"shadowCopyDirectory"
 #define CS_ASPNETCORE_CLEAN_SHADOW_DIRECTORY_CONTENT     L"cleanShadowCopyDirectory"
 #define CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG             L"disallowRotationOnConfigChange"
+#define CS_ASPNETCORE_SHUTDOWN_DELAY                     L"shutdownDelay"
+#define CS_ASPNETCORE_SHUTDOWN_DELAY_ENV                 L"ANCM_shutdownDelay"
 
 ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
         m_hostingModel(HOSTING_UNKNOWN),
@@ -53,7 +55,7 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
 
     auto disallowRotationOnConfigChange = find_element(handlerSettings, CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG).value_or(std::wstring());
     m_fDisallowRotationOnConfigChange = equals_ignore_case(L"true", disallowRotationOnConfigChange);
-                
+
     m_strProcessPath = section->GetRequiredString(CS_ASPNETCORE_PROCESS_EXE_PATH);
     m_strArguments = section->GetString(CS_ASPNETCORE_PROCESS_ARGUMENTS).value_or(CS_ASPNETCORE_PROCESS_ARGUMENTS_DEFAULT);
     m_fStdoutLogEnabled = section->GetRequiredBool(CS_ASPNETCORE_STDOUT_LOG_ENABLED);
@@ -82,4 +84,38 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     auto dotnetEnvironmentEnabled = equals_ignore_case(L"Development", dotnetEnvironment);
 
     m_fShowDetailedErrors = detailedErrorsEnabled || aspnetCoreEnvironmentEnabled || dotnetEnvironmentEnabled;
+
+    // Specifies how long to delay (in milliseconds) after IIS tells us to stop before starting the application shutdown.
+    // See StartShutdown in globalmodule to see how it's used.
+    auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
+    if (shutdownDelay.empty())
+    {
+        // Fallback to environment variable if process specific config wasn't set
+        shutdownDelay = Environment::GetEnvironmentVariableValue(CS_ASPNETCORE_SHUTDOWN_DELAY_ENV)
+            .value_or(environmentVariables[CS_ASPNETCORE_SHUTDOWN_DELAY_ENV]);
+        if (shutdownDelay.empty())
+        {
+            // Default if neither process specific config or environment variable aren't set
+            m_fShutdownDelay = std::chrono::seconds(0);
+        }
+        else
+        {
+            SetShutdownDelay(shutdownDelay);
+        }
+    }
+    else
+    {
+        SetShutdownDelay(shutdownDelay);
+    }
+}
+
+void ShimOptions::SetShutdownDelay(const std::wstring& shutdownDelay)
+{
+    auto millsecondsValue = std::stoi(shutdownDelay);
+    if (millsecondsValue < 0)
+    {
+        throw ConfigurationLoadException(format(
+            L"'shutdownDelay' in web.config or '%s' environment variable is less than 0.", CS_ASPNETCORE_SHUTDOWN_DELAY_ENV));
+    }
+    m_fShutdownDelay = std::chrono::milliseconds(millsecondsValue);
 }

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h

@@ -89,6 +89,12 @@ class ShimOptions: NonCopyable
         return m_fDisallowRotationOnConfigChange;
     }
 
+    std::chrono::milliseconds
+    QueryShutdownDelay() const noexcept
+    {
+        return m_fShutdownDelay;
+    }
+
     ShimOptions(const ConfigurationSource &configurationSource);
 
 private:
@@ -104,4 +110,7 @@ class ShimOptions: NonCopyable
     bool                           m_fCleanShadowCopyDirectory;
     bool                           m_fDisallowRotationOnConfigChange;
     std::wstring                   m_strShadowCopyingDirectory;
+    std::chrono::milliseconds      m_fShutdownDelay;
+
+    void SetShutdownDelay(const std::wstring& shutdownDelay);
 };

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp

@@ -143,22 +143,26 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             }
         }
 
-        // If we receive a request at this point.
-        // OutOfProcess: we will create a new application with new configuration
-        // InProcess: the request would have to be rejected, as we are about to call g_HttpServer->RecycleProcess
-        // on the worker process
-
         if (!applicationsToRecycle.empty())
         {
             for (auto& application : applicationsToRecycle)
             {
                 try
                 {
-                    application->ShutDownApplication(/* fServerInitiated */ false);
+                    if (UseLegacyShutdown())
+                    {
+                        application->ShutDownApplication(/* fServerInitiated */ false);
+                    }
+                    else
+                    {
+                        // Recycle the process to trigger OnGlobalStopListening
+                        // which will shutdown the server and stop listening for new requests for this app
+                        m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
+                    }
                 }
                 catch (...)
                 {
-                    LOG_ERRORF(L"Failed to stop application '%ls'", application->QueryApplicationInfoKey().c_str());
+                    LOG_ERRORF(L"Failed to recycle application '%ls'", application->QueryApplicationInfoKey().c_str());
                     OBSERVE_CAUGHT_EXCEPTION()
 
                     // Failed to recycle an application. Log an event
@@ -176,28 +180,31 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             }
         }
 
-        // Remove apps after calling shutdown on each of them
-        // This is exclusive to in-process, as the shutdown of an in-process app recycles
-        // the entire worker process.
-        if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS)
+        if (UseLegacyShutdown())
         {
-            SRWExclusiveLock lock(m_srwLock);
-            const std::wstring configurationPath = pszApplicationId;
-
-            auto itr = m_pApplicationInfoHash.begin();
-            while (itr != m_pApplicationInfoHash.end())
+            // Remove apps after calling shutdown on each of them
+            // This is exclusive to in-process, as the shutdown of an in-process app recycles
+            // the entire worker process.
+            if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS)
             {
-                if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath)
-                    && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end())
-                {
-                    itr = m_pApplicationInfoHash.erase(itr);
-                }
-                else
+                SRWExclusiveLock lock(m_srwLock);
+                const std::wstring configurationPath = pszApplicationId;
+
+                auto itr = m_pApplicationInfoHash.begin();
+                while (itr != m_pApplicationInfoHash.end())
                 {
-                    ++itr;
+                    if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath)
+                        && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end())
+                    {
+                        itr = m_pApplicationInfoHash.erase(itr);
+                    }
+                    else
+                    {
+                        ++itr;
+                    }
                 }
-            }
-        } // Release Exclusive m_srwLock
+            } // Release Exclusive m_srwLock
+        }
     }
     CATCH_RETURN()
 
@@ -211,14 +218,19 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
 VOID
 APPLICATION_MANAGER::ShutDown()
 {
+    // During shutdown we lock until we delete the application
+    SRWExclusiveLock lock(m_srwLock);
+
     // We are guaranteed to only have one outstanding OnGlobalStopListening event at a time
     // However, it is possible to receive multiple OnGlobalStopListening events
     // Protect against this by checking if we already shut down.
+    if (g_fInShutdown)
+    {
+        return;
+    }
+
     g_fInShutdown = TRUE;
     g_fInAppOfflineShutdown = true;
-
-    // During shutdown we lock until we delete the application
-    SRWExclusiveLock lock(m_srwLock);
     for (auto & [str, applicationInfo] : m_pApplicationInfoHash)
     {
         applicationInfo->ShutDownApplication(/* fServerInitiated */ true);

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h

@@ -47,6 +47,16 @@ class APPLICATION_MANAGER
         return !m_handlerResolver.GetDisallowRotationOnConfigChange();
     }
 
+    std::chrono::milliseconds GetShutdownDelay() const
+    {
+        return m_handlerResolver.GetShutdownDelay();
+    }
+
+    bool UseLegacyShutdown() const
+    {
+        return m_handlerResolver.GetShutdownDelay() == std::chrono::milliseconds::zero();
+    }
+
 private:
 
     std::unordered_map<std::wstring, std::shared_ptr<APPLICATION_INFO>>      m_pApplicationInfoHash;

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp

@@ -125,13 +125,14 @@ HRESULT
                                   moduleFactory.release(),
                                   RQ_EXECUTE_REQUEST_HANDLER,
                                   0));
-;
+
     auto pGlobalModule = std::make_unique<ASPNET_CORE_GLOBAL_MODULE>(std::move(applicationManager));
 
     RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
-                                     pGlobalModule.release(),
-                                     GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-                                     GL_STOP_LISTENING));   // worker process stop or recycle
+        pGlobalModule.release(),
+        GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
+        GL_STOP_LISTENING | // worker process will stop listening for http requests
+        GL_APPLICATION_STOP)); // app pool recycle or stop
 
     return S_OK;
 }

src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp

@@ -6,7 +6,7 @@
 extern BOOL         g_fInShutdown;
 
 ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptr<APPLICATION_MANAGER> pApplicationManager) noexcept
-    :m_pApplicationManager(std::move(pApplicationManager))
+    : m_pApplicationManager(std::move(pApplicationManager))
 {
 }
 
@@ -16,26 +16,52 @@ ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptr<APPLICATION
 //
 GLOBAL_NOTIFICATION_STATUS
 ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening(
-    _In_ IGlobalStopListeningProvider * pProvider
+    _In_ IGlobalStopListeningProvider* pProvider
 )
 {
     UNREFERENCED_PARAMETER(pProvider);
 
     LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening");
 
-    if (g_fInShutdown)
+    if (g_fInShutdown || m_shutdown.joinable())
     {
         // Avoid receiving two shutdown notifications.
         return GL_NOTIFICATION_CONTINUE;
     }
 
-    m_pApplicationManager->ShutDown();
-    m_pApplicationManager = nullptr;
+    StartShutdown();
 
     // Return processing to the pipeline.
     return GL_NOTIFICATION_CONTINUE;
 }
 
+GLOBAL_NOTIFICATION_STATUS
+ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
+    IN IHttpApplicationStopProvider* pProvider
+)
+{
+    UNREFERENCED_PARAMETER(pProvider);
+
+    // If we're already cleaned up just return.
+    // If user has opted out of the new shutdown behavior ignore this call as we never registered for it before
+    if (!m_pApplicationManager || m_pApplicationManager->UseLegacyShutdown())
+    {
+        return GL_NOTIFICATION_CONTINUE;
+    }
+
+    LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop");
+
+    if (!g_fInShutdown && !m_shutdown.joinable())
+    {
+        // Apps with preload + always running that don't receive a request before recycle/shutdown will never call OnGlobalStopListening
+        // IISExpress can also close without calling OnGlobalStopListening which is where we usually would trigger shutdown
+        // so we should make sure to shutdown the server in those cases
+        StartShutdown();
+    }
+
+    return GL_NOTIFICATION_CONTINUE;
+}
+
 //
 // Is called when configuration changed
 // Recycled the corresponding core app if its configuration changed