Open source companion commit for rstudio/rstudio-pro#248 - Spawner ch…

…anges and implementation of Local Spawner

dependencies/common/install-boost

@@ -32,7 +32,7 @@ BOOST_BUILD_DIR=boost-build
 BOOST_MODULES="algorithm asio array bind chrono circular_buffer context crc
    date_time filesystem foreach format function interprocess iostreams
    lambda lexical_cast optional program_options property_tree random range ref
-   regex scope_exit signals smart_ptr spirit string_algo system
+   regex scope_exit signals signals2 smart_ptr spirit string_algo system
    test thread tokenizer type_traits typeof unordered utility variant"
 
 # install if we aren't already installed

src/cpp/core/ProgramOptions.cpp

@@ -76,13 +76,76 @@ void reportWarnings(const std::string& warningMessages,
       core::log::logWarningMessage(warningMessages, location);
 }
 
+void parseCommandLine(variables_map& vm,
+                      const OptionsDescription& optionsDescription,
+                      const options_description& commandLineOptions,
+                      int argc,
+                      char * const argv[],
+                      std::vector<std::string>* pUnrecognized)
+{
+   // parse the command line
+   command_line_parser parser(argc, const_cast<char**>(argv));
+   parser.options(commandLineOptions);
+   parser.positional(optionsDescription.positionalOptions);
+   if (pUnrecognized != NULL)
+      parser.allow_unregistered();
+   parsed_options parsed = parser.run();
+   store(parsed, vm);
+   notify(vm) ;
+
+   // collect unrecognized if necessary
+   if (pUnrecognized != NULL)
+   {
+      *pUnrecognized = collect_unrecognized(parsed.options,
+                                            include_positional);
+   }
+}
+
+bool parseConfigFile(variables_map& vm,
+                     const std::string& configFile,
+                     const OptionsDescription& optionsDescription,
+                     bool allowUnregisteredConfigOptions)
+{
+   // open the config file
+   if (!configFile.empty())
+   {
+      boost::shared_ptr<std::istream> pIfs;
+      Error error = FilePath(configFile).open_r(&pIfs);
+      if (error)
+      {
+         reportError("Unable to open config file: " + configFile,
+                     ERROR_LOCATION);
+
+         return false;
+      }
+
+      try
+      {
+         // parse config file
+         store(parse_config_file(*pIfs, optionsDescription.configFile, allowUnregisteredConfigOptions), vm) ;
+         notify(vm) ;
+      }
+      catch(const std::exception& e)
+      {
+         reportError(
+           "Error reading " + configFile + ": " + std::string(e.what()),
+           ERROR_LOCATION);
+
+         return false;
+      }
+   }
+
+   return true;
+}
+
 
 ProgramStatus read(const OptionsDescription& optionsDescription,
                    int argc,
                    char * const argv[],
                    std::vector<std::string>* pUnrecognized,
                    bool* pHelp,
-                   bool allowUnregisteredConfigOptions)
+                   bool allowUnregisteredConfigOptions,
+                   bool configFileHasPrecedence)
 {
    *pHelp = false;
    std::string configFile;
@@ -103,58 +166,36 @@ ProgramStatus read(const OptionsDescription& optionsDescription,
       options_description commandLineOptions(optionsDescription.commandLine);
       commandLineOptions.add(general);
 
-      // parse the command line
       variables_map vm ;
-      command_line_parser parser(argc, const_cast<char**>(argv));
-      parser.options(commandLineOptions);
-      parser.positional(optionsDescription.positionalOptions);
-      if (pUnrecognized != NULL)
-         parser.allow_unregistered();
-      parsed_options parsed = parser.run();
-      store(parsed, vm);
-      notify(vm) ;
-
-      // collect unrecognized if necessary
-      if (pUnrecognized != NULL)
+
+      // the order of parsing is determined based on whether or not the config file has precedence
+      // if it does, parse it first, otherwise parse the command line first
+      if (configFileHasPrecedence)
       {
-         *pUnrecognized = collect_unrecognized(parsed.options,
-                                               include_positional);
-      }
+         // if we are parsing the config file first, do not attempt to parse
+         // the config file path from the command line arguments - just use
+         // the default value that was passed in
+         configFile = optionsDescription.defaultConfigFilePath;
 
-      // "none" is a special sentinel value for the config-file which
-      // explicitly prevents us from reading the defautl config file above
-      // now that we are past that we can reset it to empty
-      if (configFile == "none")
-         configFile = "";
-
-      // open the config file
-      if (!configFile.empty())
+         if (!parseConfigFile(vm, configFile, optionsDescription, allowUnregisteredConfigOptions))
+            return ProgramStatus::exitFailure();
+
+         parseCommandLine(vm, optionsDescription, commandLineOptions, argc, argv, pUnrecognized);
+      }
+      else
       {
-         boost::shared_ptr<std::istream> pIfs;
-         Error error = FilePath(configFile).open_r(&pIfs);
-         if (error)
-         {
-            reportError("Unable to open config file: " + configFile,
-                        ERROR_LOCATION);
-            return ProgramStatus::exitFailure() ;
-         }
-
-         try
-         {
-            // parse config file
-            store(parse_config_file(*pIfs, optionsDescription.configFile, allowUnregisteredConfigOptions), vm) ;
-            notify(vm) ;
-         }
-         catch(const std::exception& e)
-         {
-            reportError(
-              "Error reading " + configFile + ": " + std::string(e.what()),
-              ERROR_LOCATION);
+         parseCommandLine(vm, optionsDescription, commandLineOptions, argc, argv, pUnrecognized);
 
+         // "none" is a special sentinel value for the config-file which
+         // explicitly prevents us from reading the default config file above
+         // now that we are past that we can reset it to empty
+         if (configFile == "none")
+            configFile = "";
+
+         if (!parseConfigFile(vm, configFile, optionsDescription, allowUnregisteredConfigOptions))
             return ProgramStatus::exitFailure();
-         }
       }
-      
+
       // show help if requested
       if (vm.count("help"))
       {

src/cpp/core/http/RequestParser.cpp

@@ -16,6 +16,7 @@
 
 #include <core/http/RequestParser.hpp>
 
+#include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
 
 namespace rstudio {
@@ -256,7 +257,7 @@ RequestParser::status RequestParser::consume(Request& req, char input)
       state_ = space_before_header_value;
 
       // look for special content-length state
-      if ( !req.headers_.back().name.compare("Content-Length") )
+      if ( boost::iequals(req.headers_.back().name, "Content-Length") )
          parsing_content_length_ = true ;
 
       return incomplete;

src/cpp/core/http/Util.cpp

@@ -296,6 +296,14 @@ namespace {
 
 const char * const kHttpDateFormat = "%a, %d %b %Y %H:%M:%S GMT";
 const char * const kAtomDateFormat = "%Y-%m-%dT%H:%M:%S%F%Q";
+
+// facet for http date (construct w/ a_ref == 1 so we manage memory)
+// statically initialized because init is very expensive
+boost::posix_time::time_facet s_httpDateFacet(kHttpDateFormat,
+                                              boost::posix_time::time_facet::period_formatter_type(),
+                                              boost::posix_time::time_facet::special_values_formatter_type(),
+                                              boost::posix_time::time_facet::date_gen_formatter_type(),
+                                              1);
 
 boost::posix_time::ptime parseDate(const std::string& date, const char* format)
 {
@@ -331,13 +339,9 @@ std::string httpDate(const boost::posix_time::ptime& datetime)
 {
    using namespace boost::posix_time;
 
-   // facet for http date (construct w/ a_ref == 1 so we manage memory)
-   time_facet httpDateFacet(1); 
-   httpDateFacet.format("%a, %d %b %Y %H:%M:%S GMT");
-
    // output and return the date
    std::ostringstream dateStream;
-   dateStream.imbue(std::locale(dateStream.getloc(), &httpDateFacet));
+   dateStream.imbue(std::locale(dateStream.getloc(), &s_httpDateFacet));
    dateStream << datetime;
    return dateStream.str();
 }

src/cpp/core/include/core/ProgramOptions.hpp

@@ -56,34 +56,41 @@ ProgramStatus read(const OptionsDescription& optionsDescription,
                    char * const argv[],
                    std::vector<std::string>* pUnrecognized,
                    bool* pHelp,
-                   bool allowUnregisteredConfigOptions = false);
+                   bool allowUnregisteredConfigOptions = false,
+                   bool configFileHasPrecedence = false);
 
 inline ProgramStatus read(const OptionsDescription& optionsDescription,
                           int argc,
                           char * const argv[],
                           bool* pHelp,
-                          bool allowUnregisteredConfigOptions = false)
+                          bool allowUnregisteredConfigOptions = false,
+                          bool configFileHasPrecedence = false)
 {
-   return read(optionsDescription, argc, argv, NULL, pHelp, allowUnregisteredConfigOptions);
+   return read(optionsDescription, argc, argv, NULL, pHelp,
+               allowUnregisteredConfigOptions, configFileHasPrecedence);
 }
 
 inline ProgramStatus read(const OptionsDescription& optionsDescription,
                           int argc,
                           char * const argv[],
                           std::vector<std::string>* pUnrecognized,
-                          bool allowUnregisteredConfigOptions = false)
+                          bool allowUnregisteredConfigOptions = false,
+                          bool configFileHasPrecedence = false)
 {
    bool help;
-   return read(optionsDescription, argc, argv, pUnrecognized, &help, allowUnregisteredConfigOptions);
+   return read(optionsDescription, argc, argv, pUnrecognized, &help,
+               allowUnregisteredConfigOptions, configFileHasPrecedence);
 }
 
 inline ProgramStatus read(const OptionsDescription& optionsDescription,
                           int argc,
                           char * const argv[],
-                          bool allowUnregisteredConfigOptions = false)
+                          bool allowUnregisteredConfigOptions = false,
+                          bool configFileHasPrecedence = false)
 {
    bool help;
-   return read(optionsDescription, argc, argv, &help, allowUnregisteredConfigOptions);
+   return read(optionsDescription, argc, argv, &help,
+               allowUnregisteredConfigOptions, configFileHasPrecedence);
 }
 
 void reportError(const std::string& errorMessage,

src/cpp/core/include/core/json/Json.hpp

@@ -64,6 +64,8 @@ bool isType(const Value& value)
       return value.type() == IntegerType; 
    else if (boost::is_same<T, unsigned int>::value)
       return value.type() == IntegerType;
+   else if (boost::is_same<T, int64_t>::value)
+      return value.type() == IntegerType;
    else if (boost::is_same<T, unsigned long>::value)
       return value.type() == IntegerType;
    else if (boost::is_same<T, double>::value)

src/cpp/core/include/core/json/JsonRpc.hpp

@@ -1071,6 +1071,26 @@ core::Error readObject(const json::Object& object,
    return readObject(object, name12, pValue12);
 }
 
+template <typename T>
+core::Error getOptionalParam(const json::Object& json, const std::string& param,
+                             const T& defaultValue, T* outParam)
+{
+   json::Object::const_iterator it = json.find(param);
+   if (it != json.end())
+   {
+      if (!json::isType<T>(it->second))
+      {
+         return core::Error(errc::ParamTypeMismatch, ERROR_LOCATION);
+      }
+
+      *outParam = (it->second).get_value<T>();
+      return Success();
+   }
+
+   *outParam = defaultValue;
+   return Success();
+}
+
 // json rpc response
 
 class JsonRpcResponse

src/cpp/core/include/core/system/FileMode.hpp

@@ -37,6 +37,7 @@ enum FileMode
    UserReadWriteExecuteMode,
    UserReadWriteGroupReadMode,
    UserReadWriteGroupEveryoneReadMode,
+   UserReadWriteExecuteGroupEveryoneReadExecuteMode,
    EveryoneReadMode,
    EveryoneReadWriteMode,
    EveryoneReadWriteExecuteMode
@@ -65,6 +66,10 @@ inline Error changeFileMode(const FilePath& filePath,
          mode =  S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
          break;
 
+      case UserReadWriteExecuteGroupEveryoneReadExecuteMode:
+         mode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH;
+         break;
+
       case EveryoneReadMode:
          mode = S_IRUSR | S_IRGRP | S_IROTH;
          break;
@@ -140,6 +145,8 @@ inline Error getFileMode(const FilePath& filePath, FileMode* pFileMode)
       *pFileMode = EveryoneReadWriteMode;
    else if (mode == "rwxrwxrwx")
       *pFileMode = EveryoneReadWriteExecuteMode;
+   else if (mode == "rwxr-xr-x")
+      *pFileMode = UserReadWriteExecuteGroupEveryoneReadExecuteMode;
    else
        return systemError(boost::system::errc::not_supported, ERROR_LOCATION);
 

src/cpp/core/include/core/system/PosixChildProcess.hpp

@@ -16,6 +16,8 @@
 #ifndef CORE_SYSTEM_POSIX_CHILD_PROCESS_HPP
 #define CORE_SYSTEM_POSIX_CHILD_PROCESS_HPP
 
+#include <boost/asio/io_service.hpp>
+
 #include <core/system/ChildProcess.hpp>
 #include <core/system/Process.hpp>
 
@@ -70,6 +72,8 @@ class AsioAsyncChildProcess : public IAsioAsyncChildProcess, public AsyncChildPr
    virtual core::FilePath getCwd() const;
    virtual bool hasRecentOutput() const;
 
+   pid_t pid() const;
+
 private:
    struct Impl;
    boost::shared_ptr<Impl> pAsioImpl_;

src/cpp/core/include/core/system/PosixProcess.hpp

@@ -19,6 +19,7 @@
 #include <boost/asio/io_service.hpp>
 
 #include <core/system/Process.hpp>
+#include <core/system/PosixChildProcess.hpp>
 
 namespace rstudio {
 namespace core {
@@ -37,11 +38,13 @@ class AsioProcessSupervisor : boost::noncopyable
    core::Error runProgram(const std::string& executable,
                           const std::vector<std::string>& args,
                           const ProcessOptions& options,
-                          const ProcessCallbacks& callbacks);
+                          const ProcessCallbacks& callbacks,
+                          boost::shared_ptr<AsioAsyncChildProcess>* pOutChild = NULL);
 
    core::Error runCommand(const std::string& command,
                           const ProcessOptions& options,
-                          const ProcessCallbacks& callbacks);
+                          const ProcessCallbacks& callbacks,
+                          boost::shared_ptr<AsioAsyncChildProcess>* pOutChild = NULL);
 
    // Check whether any children are currently running
    bool hasRunningChildren();
@@ -56,7 +59,7 @@ class AsioProcessSupervisor : boost::noncopyable
 private:
    struct Impl;
    friend struct Impl;
-   boost::scoped_ptr<Impl> pImpl_;
+   boost::shared_ptr<Impl> pImpl_;
 };
 
 } // namespace system

src/cpp/core/include/core/system/PosixSystem.hpp

@@ -216,6 +216,11 @@ core::Error temporarilyDropPriv(const std::string& newUsername);
 core::Error permanentlyDropPriv(const std::string& newUsername);
 core::Error restorePriv();
 
+// restoreRoot should be used to set the effective ID back to root (0) before using
+// the other privilege-modifying methods above - this is necessary because they maintain
+// state of the original effective user, and in most cases that should be root
+core::Error restoreRoot();
+
 #ifdef __APPLE__
 // Detect subprocesses via Mac-only BSD-ish APIs
 std::vector<SubprocInfo> getSubprocessesMac(PidType pid);