Merge branch 'hotfix/4.1.19'

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Change Log
 
+## [4.1.19](https://github.com/TheHive-Project/TheHive/milestone/89) (2022-04-07)
+
+**Implemented enhancements:**
+
+- Migrate hive3 to 4, migrate SSL error. Requesting SSL bypass option [\#2356](https://github.com/TheHive-Project/TheHive/issues/2356)
+- [Enhancement] Improve logging [\#2371](https://github.com/TheHive-Project/TheHive/issues/2371)
+
+**Fixed bugs:**
+
+- [Bug] Regression starting with 4.1.17 in the migration tool with certificate validation  [\#2342](https://github.com/TheHive-Project/TheHive/issues/2342)
+- [Bug] Deadlock when the index backend changed [\#2351](https://github.com/TheHive-Project/TheHive/issues/2351)
+- [Bug][Security] TheHive4 libraries vulnerabilities [\#2362](https://github.com/TheHive-Project/TheHive/issues/2362)
+- [Bug] The Link given in /etc/thehive/application.conf says error. [\#2364](https://github.com/TheHive-Project/TheHive/issues/2364)
+- [Bug] An unshared case is still visible [\#2366](https://github.com/TheHive-Project/TheHive/issues/2366)
+- [Bug] Prevent custom field creation if it already exists [\#2367](https://github.com/TheHive-Project/TheHive/issues/2367)
+- [Bug] An user may exist without being member of any organisation [\#2368](https://github.com/TheHive-Project/TheHive/issues/2368)
+- [Bug] Tags can't be updated [\#2369](https://github.com/TheHive-Project/TheHive/issues/2369)
+- [Bug] Dashboarding on weeks provide incorrect results on specific dates [\#2370](https://github.com/TheHive-Project/TheHive/issues/2370)
+
 ## [4.1.18](https://github.com/TheHive-Project/TheHive/milestone/88) (2022-02-07)
 
 **Implemented enhancements:**

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.18-1"
+val thehiveVersion         = "4.1.19-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -62,11 +62,15 @@ libraryDependencies in ThisBuild ++= {
   }
 }
 dependencyOverrides in ThisBuild ++= Seq(
-//  "org.locationtech.spatial4j" % "spatial4j"                 % "0.6",
-//  "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2
   akkaActor,
   logbackClassic
 )
+val securityUpdates = Seq(
+  "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.6",
+  "com.fasterxml.jackson.core"    % "jackson-databind"     % "2.12.6.1",
+  "org.yaml"                      % "snakeyaml"            % "1.30"
+)
+dependencyOverrides in ThisBuild ++= securityUpdates
 PlayKeys.includeDocumentationInBinary := false
 milestoneFilter := ((milestone: Milestone) => milestone.title.startsWith("4"))
 

conf/application.sample.conf

@@ -1,5 +1,5 @@
 ###
-## Documentation is available at https://github.com/TheHive-Project/TheHiveDocs/TheHive4
+## Documentation is available at https://docs.thehive-project.org/thehive/
 ###
 
 ## Include Play secret key

conf/logback.xml

@@ -15,13 +15,13 @@
             <maxFileSize>10MB</maxFileSize>
         </triggeringPolicy>
         <encoder>
-            <pattern>%date [%level] from %logger in %thread [%X{request}|%X{tx}] %message%n%xException</pattern>
+            <pattern>%date [%level] from %logger in %thread %replace(\(%X{userId}@%X{organisation}\) ){'\(@\) ',''}[%X{request}|%X{tx}] %message%n%xException</pattern>
         </encoder>
     </appender>
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%coloredLevel %logger{15} [%X{request}|%X{tx}] %message%n%xException{10}
+            <pattern>%coloredLevel %logger{15} %replace(\(%X{userId}@%X{organisation}\) ){'\(@\) ',''}[%X{request}|%X{tx}] %message%n%xException{10}
             </pattern>
         </encoder>
     </appender>

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala

@@ -58,7 +58,7 @@ class JobCtrl @Inject() (
           db.roTransaction { implicit graph =>
             val artifactId: String = request.body("artifactId")
             for {
-              o <- observableSrv.get(EntityIdOrName(artifactId)).richObservable.getOrFail("Observable")
+              o <- observableSrv.get(EntityIdOrName(artifactId)).can(Permissions.manageAnalyse).richObservable.getOrFail("Observable")
               c <- observableSrv.get(EntityIdOrName(artifactId)).`case`.getOrFail("Case")
             } yield (o, c)
           }.fold(

frontend/app/scripts/controllers/admin/attack/AttackPatternListCtrl.js

@@ -163,7 +163,7 @@
                 .catch(function (response) {
                     NotificationSrv.error('AttackPatternImportCtrl', response.data, response.status);
                 })
-                .fincally(function () {
+                .finally(function () {
                     this.loading = false;
                 });
         };

frontend/app/views/partials/observables/list/observables.html

@@ -39,7 +39,7 @@ <h4>
                     </a>
                 </th>
                 <th>
-                    <a href class="text-default ml-xxxs" ng-click="sortByField('data')" uib-tooltip="Sort by dataType">
+                    <a href class="text-default ml-xxxs" ng-click="sortByField('data')" uib-tooltip="Sort by data">
                         Value/Filename
                         <i ng-show="filtering.context.sort.indexOf('+data') === -1 && filtering.context.sort.indexOf('-data') === -1"
                             class="fa fa-sort"></i>

frontend/bower.json

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.18-1",
+  "version": "4.1.19-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.18-1",
+    "version": "4.1.19-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

@@ -5,7 +5,7 @@ import akka.actor.typed.ActorRef
 import akka.stream.Materializer
 import com.google.inject.{Guice, Injector => GInjector}
 import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder}
-import org.thp.scalligraph.auth.{AuthContext, UserSrv => UserDB}
+import org.thp.scalligraph.auth.{UserSrv => UserDB}
 import org.thp.scalligraph.models.Database
 import org.thp.scalligraph.services._
 import org.thp.thehive.migration.th4.DummyActor
@@ -21,7 +21,6 @@ import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
-import scala.util.Success
 
 trait IntegrityCheckApp {
   private def buildApp(configuration: Configuration, db: Database)(implicit actorSystem: ActorSystem): GInjector =
@@ -78,9 +77,8 @@ trait IntegrityCheckApp {
     buildApp(configuration, db).getInstance(classOf[IntegrityChecks]).runChecks()
 }
 
-class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[IntegrityCheck], userSrv: UserDB) extends MapMerger {
-  def runChecks(): Unit = {
-    implicit val authContext: AuthContext = userSrv.getSystemAuthContext
+class IntegrityChecks @Inject() (checks: immutable.Set[IntegrityCheck]) extends MapMerger {
+  def runChecks(): Unit =
     checks.foreach { c =>
       println(s"Running check on ${c.name} ...")
       val desupStats = c match {
@@ -102,5 +100,4 @@ class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[IntegrityCh
         println(s"  $statsStr")
 
     }
-  }
 }

migration/src/main/scala/org/thp/thehive/migration/th3/ElasticClient.scala

@@ -5,31 +5,35 @@ import akka.actor.{ActorSystem, Scheduler}
 import akka.stream.Materializer
 import akka.stream.scaladsl.{Sink, Source}
 import akka.util.ByteString
-import com.typesafe.sslconfig.ssl.{KeyManagerConfig, KeyStoreConfig, SSLConfigSettings, TrustManagerConfig, TrustStoreConfig}
-import org.thp.client.{Authentication, NoAuthentication, PasswordAuthentication}
+import com.typesafe.config.ConfigRenderOptions
+import com.typesafe.sslconfig.ssl.{KeyManagerConfig, KeyStoreConfig, TrustManagerConfig, TrustStoreConfig}
+import org.thp.client._
+import org.thp.scalligraph.utils.FunctionalCondition.When
 import org.thp.scalligraph.utils.Retry
 import org.thp.scalligraph.{InternalError, NotFoundError}
 import play.api.http.HeaderNames
 import play.api.libs.json.{JsNumber, JsObject, JsValue, Json}
-import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfig}
-import play.api.libs.ws.{WSClient, WSClientConfig, WSResponse}
+import play.api.libs.ws.ahc.AhcWSClientConfig
+import play.api.libs.ws.{WSClient, WSResponse}
 import play.api.{Configuration, Logger}
 
 import java.net.{URI, URLEncoder}
 import javax.inject.{Inject, Provider, Singleton}
-import scala.concurrent.duration.{Duration, DurationInt, DurationLong, FiniteDuration}
+import scala.concurrent.duration.{DurationInt, DurationLong, FiniteDuration}
 import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.Try
 
 @Singleton
 class ElasticClientProvider @Inject() (
     config: Configuration,
+    mat: Materializer,
     implicit val actorSystem: ActorSystem
 ) extends Provider[ElasticClient] {
 
   override def get(): ElasticClient = {
     lazy val logger = Logger(getClass)
     val ws: WSClient = {
+
       val trustManager = config.getOptional[String]("search.trustStore.path").map { trustStore =>
         val trustStoreConfig = TrustStoreConfig(None, Some(trustStore))
         config.getOptional[String]("search.trustStore.type").foreach(trustStoreConfig.withStoreType)
@@ -46,33 +50,40 @@ class ElasticClientProvider @Inject() (
         keyManager.withKeyStoreConfigs(List(keyStoreConfig))
         keyManager
       }
-      val sslConfig = SSLConfigSettings()
-      trustManager.foreach(sslConfig.withTrustManagerConfig)
-      keyManager.foreach(sslConfig.withKeyManagerConfig)
+      val connectionTimeout    = config.getOptional[Int]("search.connectTimeout").map(_.millis)
+      val idleTimeout          = config.getOptional[Int]("search.socketTimeout").map(_.millis)
+      val requestTimeout       = config.getOptional[Int]("search.connectionRequestTimeout").map(_.millis)
+      val followRedirects      = config.getOptional[Boolean]("search.redirectsEnabled")
+      val maxNumberOfRedirects = config.getOptional[Int]("search.maxRedirects")
 
-      val wsConfig = AhcWSClientConfig(
-        wsClientConfig = WSClientConfig(
-          connectionTimeout = config.getOptional[Int]("search.connectTimeout").fold(2.minutes)(_.millis),
-          idleTimeout = config.getOptional[Int]("search.socketTimeout").fold(2.minutes)(_.millis),
-          requestTimeout = config.getOptional[Int]("search.connectionRequestTimeout").fold(2.minutes)(_.millis),
-          followRedirects = config.getOptional[Boolean]("search.redirectsEnabled").getOrElse(false),
-          useProxyProperties = true,
-          userAgent = None,
-          compressionEnabled = false,
-          ssl = sslConfig
-        ),
-        maxConnectionsPerHost = -1,
-        maxConnectionsTotal = -1,
-        maxConnectionLifetime = Duration.Inf,
-        idleConnectionInPoolTimeout = 1.minute,
-        maxNumberOfRedirects = config.getOptional[Int]("search.maxRedirects").getOrElse(5),
-        maxRequestRetry = 5,
-        disableUrlEncoding = false,
-        keepAlive = true,
-        useLaxCookieEncoder = false,
-        useCookieStore = false
-      )
-      AhcWSClient(wsConfig)
+      val wsConfig = Try(Json.parse(config.underlying.getValue("search.trustStore.wsConfig").render(ConfigRenderOptions.concise())).as[ProxyWSConfig])
+        .getOrElse(ProxyWSConfig(AhcWSClientConfig(), None))
+        .merge(trustManager) { (cfg, tm) =>
+          cfg.copy(wsConfig =
+            cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(ssl = cfg.wsConfig.wsClientConfig.ssl.withTrustManagerConfig(tm)))
+          )
+        }
+        .merge(keyManager) { (cfg, km) =>
+          cfg.copy(wsConfig =
+            cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(ssl = cfg.wsConfig.wsClientConfig.ssl.withKeyManagerConfig(km)))
+          )
+        }
+        .merge(connectionTimeout) { (cfg, ct) =>
+          cfg.copy(wsConfig = cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(connectionTimeout = ct)))
+        }
+        .merge(idleTimeout) { (cfg, it) =>
+          cfg.copy(wsConfig = cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(idleTimeout = it)))
+        }
+        .merge(requestTimeout) { (cfg, rt) =>
+          cfg.copy(wsConfig = cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(requestTimeout = rt)))
+        }
+        .merge(followRedirects) { (cfg, fr) =>
+          cfg.copy(wsConfig = cfg.wsConfig.copy(wsClientConfig = cfg.wsConfig.wsClientConfig.copy(followRedirects = fr)))
+        }
+        .merge(maxNumberOfRedirects) { (cfg, mr) =>
+          cfg.copy(wsConfig = cfg.wsConfig.copy(maxNumberOfRedirects = mr))
+        }
+      new ProxyWS(wsConfig, mat)
     }
 
     val authentication: Authentication =

project/Dependencies.scala

@@ -32,7 +32,7 @@ object Dependencies {
   lazy val macroParadise           = "org.scalamacros"             % "paradise"                           % "2.1.1" cross CrossVersion.full
   lazy val chimney                 = "io.scalaland"               %% "chimney"                            % "0.6.1"
   lazy val reflections             = "org.reflections"             % "reflections"                        % "0.9.12"
-  lazy val hadoopClient            = "org.apache.hadoop"           % "hadoop-client"                      % "3.3.0" exclude ("log4j", "log4j")
+  lazy val hadoopClient            = "org.apache.hadoop"           % "hadoop-client"                      % "3.3.2" exclude ("log4j", "log4j")
   lazy val zip4j                   = "net.lingala.zip4j"           % "zip4j"                              % "2.6.4"
   lazy val alpakka                 = "com.lightbend.akka"         %% "akka-stream-alpakka-json-streaming" % "2.0.2"
   lazy val handlebars              = "com.github.jknack"           % "handlebars"                         % "4.2.0"

project/plugins.sbt

@@ -1,3 +1,3 @@
-addSbtPlugin("com.typesafe.play"   % "sbt-plugin"           % "2.8.7")
+addSbtPlugin("com.typesafe.play"   % "sbt-plugin"           % "2.8.13")
 addSbtPlugin("org.scalameta"       % "sbt-scalafmt"         % "2.3.0")
 addSbtPlugin("org.thehive-project" % "sbt-github-changelog" % "0.3.0")

thehive/app/org/thp/thehive/controllers/v1/ShareCtrl.scala

@@ -167,7 +167,7 @@ class ShareCtrl @Inject() (
     else if (shareSrv.get(shareId).has(_.owner, true).exists)
       Failure(AuthorizationError("You can't remove initial shares"))
     else
-      shareSrv.get(shareId).getOrFail("Share").flatMap(shareSrv.delete(_))
+      shareSrv.unshareCase(shareId)
 
   def updateShare(shareId: String): Action[AnyContent] =
     entrypoint("update share")

thehive/app/org/thp/thehive/services/AlertSrv.scala

@@ -110,7 +110,7 @@ class AlertSrv @Inject() (
       tagsToAdd <- (tags -- alert.tags).toTry(tagSrv.getOrCreate)
       tagsToRemove = get(alert).tags.toSeq.filterNot(t => tags.contains(t.toString))
       _ <- tagsToAdd.toTry(alertTagSrv.create(AlertTag(), alert, _))
-      _ = if (tags.nonEmpty) get(alert).outE[AlertTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
+      _ = if (tagsToRemove.nonEmpty) get(alert).outE[AlertTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
       _ <- get(alert)
         .update(_.tags, tags.toSeq)
         .update(_._updatedAt, Some(new Date))

thehive/app/org/thp/thehive/services/CaseSrv.scala

@@ -126,8 +126,8 @@ class CaseSrv @Inject() (
     val uniqueFields = caseTemplateCf.filter {
       case InputCustomFieldValue(name, _, _) => !caseCf.exists(_.name == name)
     }
-    (caseCf ++ uniqueFields)
-      .sortBy(cf => (cf.order.isEmpty, cf.order))
+
+    (caseCf.sortBy(_.order) ++ uniqueFields.sortBy(_.order))
       .zipWithIndex
       .map { case (InputCustomFieldValue(name, value, _), i) => InputCustomFieldValue(name, value, Some(i)) }
   }

thehive/app/org/thp/thehive/services/CaseTemplateSrv.scala

@@ -97,7 +97,7 @@ class CaseTemplateSrv @Inject() (
       tagsToAdd <- (tags -- caseTemplate.tags).toTry(tagSrv.getOrCreate)
       tagsToRemove = get(caseTemplate).tags.toSeq.filterNot(t => tags.contains(t.toString))
       _ <- tagsToAdd.toTry(caseTemplateTagSrv.create(CaseTemplateTag(), caseTemplate, _))
-      _ = if (tags.nonEmpty) get(caseTemplate).outE[CaseTemplateTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
+      _ = if (tagsToRemove.nonEmpty) get(caseTemplate).outE[CaseTemplateTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
       _ <- get(caseTemplate)
         .update(_.tags, tags.toSeq)
         .update(_._updatedAt, Some(new Date))

thehive/app/org/thp/thehive/services/CustomFieldSrv.scala

@@ -2,14 +2,13 @@ package org.thp.thehive.services
 
 import akka.actor.typed.ActorRef
 import org.apache.tinkerpop.gremlin.structure.Edge
-import org.thp.scalligraph.EntityIdOrName
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.{Database, Entity}
 import org.thp.scalligraph.query.PropertyUpdater
-import org.thp.scalligraph.RichSeq
-import org.thp.scalligraph.services.{DedupCheck, IntegrityCheckOps, VertexSrv}
+import org.thp.scalligraph.services.{DedupCheck, VertexSrv}
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal._
+import org.thp.scalligraph.{CreateError, EntityIdOrName, RichSeq}
 import org.thp.thehive.controllers.v1.Conversion._
 import org.thp.thehive.models._
 import org.thp.thehive.services.CustomFieldOps._
@@ -18,7 +17,7 @@ import play.api.libs.json.{JsObject, JsValue}
 
 import java.util.{Map => JMap}
 import javax.inject.{Inject, Provider, Singleton}
-import scala.util.{Success, Try}
+import scala.util.{Failure, Try}
 
 @Singleton
 class CustomFieldSrv @Inject() (
@@ -37,10 +36,13 @@ class CustomFieldSrv @Inject() (
   }
 
   def create(e: CustomField)(implicit graph: Graph, authContext: AuthContext): Try[CustomField with Entity] =
-    for {
-      created <- createEntity(e)
-      _       <- auditSrv.customField.create(created, created.toJson)
-    } yield created
+    if (startTraversal.getByName(e.name).exists)
+      Failure(CreateError(s"CustomField ${e.name} already exists"))
+    else
+      for {
+        created <- createEntity(e)
+        _       <- auditSrv.customField.create(created, created.toJson)
+      } yield created
 
   override def exists(e: CustomField)(implicit graph: Graph): Boolean = startTraversal.getByName(e.name).exists
 

thehive/app/org/thp/thehive/services/ObservableSrv.scala

@@ -133,7 +133,7 @@ class ObservableSrv @Inject() (
       tagsToAdd <- (tags -- observable.tags).toTry(tagSrv.getOrCreate)
       tagsToRemove = get(observable).tags.toSeq.filterNot(t => tags.contains(t.toString))
       _ <- tagsToAdd.toTry(observableTagSrv.create(ObservableTag(), observable, _))
-      _ = if (tags.nonEmpty) get(observable).outE[ObservableTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
+      _ = if (tagsToRemove.nonEmpty) get(observable).outE[ObservableTag].filter(_.otherV.hasId(tagsToRemove.map(_._id): _*)).remove()
       _ <- get(observable)
         .update(_.tags, tags.toSeq)
         .update(_._updatedAt, Some(new Date))

thehive/app/org/thp/thehive/services/UserSrv.scala

@@ -353,6 +353,7 @@ class UserIntegrityCheck @Inject() (
       (_.out("UserRole"), _.in("UserRole")),
       (_.out("RoleOrganisation"), _.in("RoleOrganisation"))
     ).flatMap(ElementSelector.firstCreatedElement(_)).map(e => removeVertices(e._2)).size
-    Map("duplicateRoleLinks" -> duplicateRoleLinks.toLong)
+    val orphanCount = service.startTraversal.filterNot(_.organisations).sideEffect(_.drop()).getCount
+    Map("duplicateRoleLinks" -> duplicateRoleLinks.toLong, "orphan" -> orphanCount)
   }
 }