Merge pull request #14649 from rstudio/bugfix-local-stream-perm-denied

Fix for very intermittent error when connecting using local stream
rstudio · May 3, 2024 · ed38750 · ed38750
2 parents ce04a06 + 94fa459
commit ed38750
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 2 deletions.
diff --git a/src/cpp/core/include/core/http/AsyncClient.hpp b/src/cpp/core/include/core/http/AsyncClient.hpp
@@ -353,12 +353,18 @@ class AsyncClient :
    virtual void connectAndWriteRequest() = 0;
    virtual std::string getDefaultHostHeader() = 0;
 
+   // A hook for LocalStreamAsyncClient connections to retry on permission denied errors that show up intermittently
+   virtual bool recentConnectionError(const Error& connectionError)
+   {
+      return false;
+   }
+
    bool retryConnectionIfRequired(const Error& connectionError,
                                   Error* pOtherError)
    {
       // retry if this is a connection unavailable error and the
       // caller has provided a connection retry profile
-      if (http::isConnectionUnavailableError(connectionError) &&
+      if ((http::isConnectionUnavailableError(connectionError) || recentConnectionError(connectionError)) &&
           !connectionRetryContext_.profile.empty())
       {
          // if this is our first retry then set our stop trying time

diff --git a/src/cpp/core/include/core/http/LocalStreamAsyncClient.hpp b/src/cpp/core/include/core/http/LocalStreamAsyncClient.hpp
@@ -49,7 +49,8 @@ class LocalStreamAsyncClient
                                                                 logToStderr),
        socket_(ioService),
        localStreamPath_(localStreamPath),
-       validateUid_(validateUid)
+       validateUid_(validateUid),
+       retriedPermDenied_(false)
    {
       setConnectionRetryProfile(retryProfile);
    }
@@ -144,10 +145,32 @@ class LocalStreamAsyncClient
       return boost::static_pointer_cast<LocalStreamAsyncClient>(ptrShared);
    }
 
+   virtual bool recentConnectionError(const Error& connectionError)
+   {
+      bool permDeniedError = connectionError.getCode() == boost::system::errc::permission_denied;
+      if (!permDeniedError)
+         return false;
+      if (retriedPermDenied_) // Already been here once so it is probably a real perm denied error
+      {
+         LOG_DEBUG_MESSAGE("Not retrying local stream connection: " + localStreamPath_.getAbsolutePath() + " - 2nd permission denied error for same connection");
+         return false;
+      }
+
+      // Seen once in 5000 sessions during load tests. Possibly the local stream socket is in a weird state
+      // for an instant as it's being created by the remote side. So the fix here is to retry on this error just
+      // once. Don't wait a full timeout interval (possibly 10-45s) if it's a real permission denied error just in case
+      // that happens regularly.
+      LOG_DEBUG_MESSAGE("Retrying local stream connection: " + localStreamPath_.getAbsolutePath() + " due to permission denied error");
+
+      retriedPermDenied_ = true;
+      return true;
+   }
+
 private:
    boost::asio::local::stream_protocol::socket socket_;
    core::FilePath localStreamPath_;
    boost::optional<UidType> validateUid_;
+   bool retriedPermDenied_;
 };