Merge branch 'hotfix/4.1.16'

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Change Log
 
+## [4.1.16](https://github.com/TheHive-Project/TheHive/milestone/86) (2021-12-17)
+
+**Implemented enhancements:**
+
+- [Feature Request] Remove persistent filters on "Similar Cases" tab [\#2282](https://github.com/TheHive-Project/TheHive/issues/2282)
+- [Enhancement] When observable data is too big, use hash [\#2288](https://github.com/TheHive-Project/TheHive/issues/2288)
+- Remove unnecessary log4j dependency  [\#2291](https://github.com/TheHive-Project/TheHive/issues/2291)
+
+**Fixed bugs:**
+
+- [Bug] Index fails with immense terms [\#2289](https://github.com/TheHive-Project/TheHive/issues/2289)
+- [Bug] Marking an alert as read do not update it's "updatedAt" nor "updatedBy" field [\#2292](https://github.com/TheHive-Project/TheHive/issues/2292)
+
 ## [4.1.15](https://github.com/TheHive-Project/TheHive/milestone/85) (2021-12-06)
 
 **Implemented enhancements:**

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.15-1"
+val thehiveVersion         = "4.1.16-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -63,8 +63,9 @@ libraryDependencies in ThisBuild ++= {
 }
 dependencyOverrides in ThisBuild ++= Seq(
 //  "org.locationtech.spatial4j" % "spatial4j"                 % "0.6",
-//  "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2"
-  akkaActor
+//  "org.elasticsearch.client" % "elasticsearch-rest-client" % "6.7.2
+  akkaActor,
+  logbackClassic
 )
 PlayKeys.includeDocumentationInBinary := false
 milestoneFilter := ((milestone: Milestone) => milestone.title.startsWith("4"))

frontend/app/scripts/components/alert/AlertSimilarCaseListCmp.js

@@ -39,15 +39,16 @@
                     this.filtering = new FilteringSrv('case', 'alert.dialog.similar-cases', {
                         version: 'v1',
                         defaults: {
-                            showFilters: true,
+                            showFilters: false,
                             showStats: false,
                             pageSize: 2,
                             sort: ['-startDate']
                         },
                         defaultFilter: []
                     });
 
-                    self.filtering.initContext('alert.dialog.similar-cases')
+                    //self.filtering.initContext('alert.dialog.similar-cases')
+                    self.filtering.initContext()
                         .then(function () {
                             var defaultFilter = AlertingSrv.getSimilarityFilter(self.state.defaultAlertSimilarCaseFilter);
 

frontend/app/views/partials/admin/platform/status.html

@@ -61,7 +61,7 @@ <h4 class="text-primary mt-m">Data index status <small class="ml-m clickable"
                             <a href class="text-primary mh-s" ng-click="$vm.reindex(index.name)">
                                 <i class="fa fa-cog mr-xxxs"></i> Reindex the data
                             </a>
-                            <a href class="text-danger" ng-click="$vm.rebuildIndex(index.name)">
+                            <a href class="text-danger" ng-click="$vm.rebuildIndex('all')">
                                 <i class="fa fa-trash mr-xxxs"></i> Drop and rebuild the index
                             </a>
 

frontend/bower.json

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.15-1",
+  "version": "4.1.16-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/package.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.15-1",
+    "version": "4.1.16-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala

@@ -630,9 +630,11 @@ class Output @Inject() (
       } yield IdMapping(inputLog.metaData.id, log._id)
     }
 
-  private def getData(value: String)(implicit graph: Graph, authContext: AuthContext): Try[Data with Entity] =
-    if (observableDataIsIndexed) dataSrv.create(Data(value))
-    else dataSrv.createEntity(Data(value))
+  private def getData(value: String)(implicit graph: Graph, authContext: AuthContext): Try[Data with Entity] = {
+    val (dataOrHash, fullData) = UseHashToIndex.hashToIndex(value).fold[(String, Option[String])](value -> None)(_ -> Some(value))
+    if (observableDataIsIndexed) dataSrv.create(Data(dataOrHash, fullData))
+    else dataSrv.createEntity(Data(dataOrHash, fullData))
+  }
 
   private def createSimpleObservable(observable: Observable, observableType: ObservableType with Entity, dataValue: String)(implicit
       graph: Graph,
@@ -700,7 +702,8 @@ class Output @Inject() (
         richObservable <- createObservable(caseId, inputObservable, organisations.map(_._id).toSet)
         _              <- reportTagSrv.updateTags(richObservable, inputObservable.reportTags)
         case0          <- getCase(caseId)
-        _              <- organisations.toTry(o => shareSrv.shareObservable(RichObservable(richObservable, None, None, Nil), case0, o._id))
+        // the data in richObservable is not set because it is not used in shareSrv
+        _ <- organisations.toTry(o => shareSrv.shareObservable(RichObservable(richObservable, None, None, None, Nil), case0, o._id))
       } yield IdMapping(inputObservable.metaData.id, richObservable._id)
     }
 

project/Dependencies.scala

@@ -9,6 +9,7 @@ object Dependencies {
   lazy val playLogback             = "com.typesafe.play"        %% "play-logback"                       % play.core.PlayVersion.current
   lazy val playGuice               = "com.typesafe.play"        %% "play-guice"                         % play.core.PlayVersion.current
   lazy val playFilters             = "com.typesafe.play"        %% "filters-helpers"                    % play.core.PlayVersion.current
+  lazy val logbackClassic          = "ch.qos.logback"            % "logback-classic"                    % "1.2.8"
   lazy val playMockws              = "de.leanovate.play-mockws" %% "play-mockws"                        % "2.8.0"
   lazy val akkaActor               = "com.typesafe.akka"        %% "akka-actor"                         % akkaVersion
   lazy val akkaCluster             = "com.typesafe.akka"        %% "akka-cluster"                       % akkaVersion
@@ -35,7 +36,7 @@ object Dependencies {
   lazy val elastic4sHttpStreams    = "com.sksamuel.elastic4s"   %% "elastic4s-http-streams"             % elastic4sVersion
   lazy val elastic4sClient         = "com.sksamuel.elastic4s"   %% "elastic4s-client-esjava"            % elastic4sVersion
   lazy val reflections             = "org.reflections"           % "reflections"                        % "0.9.12"
-  lazy val hadoopClient            = "org.apache.hadoop"         % "hadoop-client"                      % "3.3.0"
+  lazy val hadoopClient            = "org.apache.hadoop"         % "hadoop-client"                      % "3.3.0" exclude ("log4j", "log4j")
   lazy val zip4j                   = "net.lingala.zip4j"         % "zip4j"                              % "2.6.4"
   lazy val alpakka                 = "com.lightbend.akka"       %% "akka-stream-alpakka-json-streaming" % "2.0.2"
   lazy val handlebars              = "com.github.jknack"         % "handlebars"                         % "4.2.0"

thehive/app/org/thp/thehive/TheHiveModule.scala

@@ -6,11 +6,11 @@ import com.google.inject.AbstractModule
 import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder}
 import org.thp.scalligraph.SingleInstance
 import org.thp.scalligraph.auth._
-import org.thp.scalligraph.janus.JanusDatabaseProvider
+import org.thp.scalligraph.janus.{ImmenseTermProcessor, JanusDatabaseProvider}
 import org.thp.scalligraph.models.{Database, UpdatableSchema}
 import org.thp.scalligraph.services.{GenIntegrityCheckOps, HadoopStorageSrv, S3StorageSrv}
 import org.thp.thehive.controllers.v0.QueryExecutorVersion0Provider
-import org.thp.thehive.models.TheHiveSchemaDefinition
+import org.thp.thehive.models.{TheHiveSchemaDefinition, UseHashToIndex}
 import org.thp.thehive.services.notification.notifiers._
 import org.thp.thehive.services.notification.triggers._
 import org.thp.thehive.services.{UserSrv => _, _}
@@ -112,6 +112,8 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte
     bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider]
 
     bind[SingleInstance].to[ClusterSetup].asEagerSingleton()
+
+    ImmenseTermProcessor.registerStrategy("observableHashToIndex", _ => UseHashToIndex)
     ()
   }
 }

thehive/app/org/thp/thehive/controllers/v0/Conversion.scala

@@ -50,6 +50,8 @@ object Conversion {
       .withFieldComputed(_.customFields, rc => JsObject(rc.customFields.map(cf => cf.name -> Json.obj(cf.typeName -> cf.toJson))))
       .withFieldRenamed(_._createdAt, _.createdAt)
       .withFieldRenamed(_._createdBy, _.createdBy)
+      .withFieldRenamed(_._updatedAt, _.updatedAt)
+      .withFieldRenamed(_._updatedBy, _.updatedBy)
       .withFieldComputed(_._id, _._id.toString)
       .withFieldComputed(_.id, _._id.toString)
       .withFieldComputed(_.id, _._id.toString)
@@ -81,6 +83,8 @@ object Conversion {
         .withFieldComputed(_.id, _._id.toString)
         .withFieldRenamed(_._createdAt, _.createdAt)
         .withFieldRenamed(_._createdBy, _.createdBy)
+        .withFieldRenamed(_._updatedAt, _.updatedAt)
+        .withFieldRenamed(_._updatedBy, _.updatedBy)
         .withFieldConst(_._type, "alert")
         .withFieldComputed(_.tags, _.tags.toSet)
         .withFieldComputed(_.`case`, _.caseId.map(_.toString))

thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala

@@ -2,13 +2,16 @@ package org.thp.thehive.controllers.v0
 
 import net.lingala.zip4j.ZipFile
 import net.lingala.zip4j.model.FileHeader
+import org.apache.tinkerpop.gremlin.process.traversal.Compare
 import org.thp.scalligraph._
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers._
 import org.thp.scalligraph.models.{Database, Entity, UMapping}
+import org.thp.scalligraph.query.PredicateOps.PredicateOpsDefs
 import org.thp.scalligraph.query._
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{IteratorOutput, Traversal}
+import org.thp.scalligraph.utils.Hasher
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.dto.v0.{InputAttachment, InputObservable}
 import org.thp.thehive.models._
@@ -445,7 +448,15 @@ class PublicObservable @Inject() (
         _ <- observableSrv.updateType(observable, newDataType)(graph, authContext)
       } yield Json.obj("dataType" -> value)
     })
-    .property("data", UMapping.string.optional)(_.field.readonly)
+    .property("data", UMapping.string.optional)(
+      _.select(_.value(_.data))
+        .filter[String] {
+          case (_, observables, _, Right(predicate)) => observables.has(_.data, predicate.mapValue(v => UseHashToIndex.hashToIndex(v).getOrElse(v)))
+          case (_, observables, _, Left(true))       => observables.has(_.data)
+          case (_, observables, _, Left(false))      => observables.hasNot(_.data)
+        }
+        .readonly
+    )
     .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly)
     .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly)
     .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly)

thehive/app/org/thp/thehive/controllers/v1/Properties.scala

@@ -1,11 +1,13 @@
 package org.thp.thehive.controllers.v1
 
+import org.apache.tinkerpop.gremlin.process.traversal.Compare
 import org.apache.tinkerpop.gremlin.structure.T
 import org.thp.scalligraph.controllers.{FPathElem, FPathEmpty, FString}
 import org.thp.scalligraph.models.{Database, UMapping}
 import org.thp.scalligraph.query.PredicateOps._
 import org.thp.scalligraph.query.{PublicProperties, PublicPropertyListBuilder}
 import org.thp.scalligraph.traversal.TraversalOps._
+import org.thp.scalligraph.utils.Hasher
 import org.thp.scalligraph.{BadRequestError, EntityId, EntityIdOrName, InvalidFormatAttributeError, RichSeq}
 import org.thp.thehive.dto.v1.InputCustomFieldValue
 import org.thp.thehive.models._
@@ -28,7 +30,7 @@ import org.thp.thehive.services._
 import play.api.libs.json.{JsObject, JsValue, Json}
 
 import javax.inject.{Inject, Singleton}
-import scala.util.{Failure, Success, Try}
+import scala.util.{Failure, Success}
 
 @Singleton
 class Properties @Inject() (
@@ -452,7 +454,15 @@ class Properties @Inject() (
           _ <- observableSrv.updateType(observable, newDataType)(graph, authContext)
         } yield Json.obj("dataType" -> value)
       })
-      .property("data", UMapping.string.optional)(_.field.readonly)
+      .property("data", UMapping.string.optional)(
+        _.select(_.value(_.data))
+          .filter[String] {
+            case (_, observables, _, Right(predicate)) => observables.has(_.data, predicate.mapValue(v => UseHashToIndex.hashToIndex(v).getOrElse(v)))
+            case (_, observables, _, Left(true))       => observables.has(_.data)
+            case (_, observables, _, Left(false))      => observables.hasNot(_.data)
+          }
+          .readonly
+      )
       .property("attachment.name", UMapping.string.optional)(_.select(_.attachments.value(_.name)).readonly)
       .property("attachment.hashes", UMapping.hash.sequence)(_.select(_.attachments.value(_.hashes)).readonly)
       .property("attachment.size", UMapping.long.optional)(_.select(_.attachments.value(_.size)).readonly)

thehive/app/org/thp/thehive/models/Observable.scala

@@ -1,6 +1,9 @@
 package org.thp.thehive.models
 
-import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType}
+import org.apache.tinkerpop.gremlin.structure.{Vertex, VertexProperty}
+import org.thp.scalligraph.janus.{ImmenseStringTermFilter, ImmenseTermProcessor}
+import org.thp.scalligraph.models.{DefineIndex, Entity, IndexType, UMapping}
+import org.thp.scalligraph.utils.Hasher
 import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId}
 
 import java.util.Date
@@ -46,6 +49,7 @@ case class Observable(
 
 case class RichObservable(
     observable: Observable with Entity,
+    fullData: Option[Data with Entity],
     attachment: Option[Attachment with Entity],
     seen: Option[Boolean],
     reportTags: Seq[ReportTag with Entity]
@@ -60,12 +64,50 @@ case class RichObservable(
   def ioc: Boolean                                             = observable.ioc
   def sighted: Boolean                                         = observable.sighted
   def ignoreSimilarity: Option[Boolean]                        = observable.ignoreSimilarity
-  def dataOrAttachment: Either[String, Attachment with Entity] = observable.data.toLeft(attachment.get)
+  def dataOrAttachment: Either[String, Attachment with Entity] = data.toLeft(attachment.get)
   def dataType: String                                         = observable.dataType
-  def data: Option[String]                                     = observable.data
+  def data: Option[String]                                     = fullData.map(d => d.fullData.getOrElse(d.data))
   def tags: Seq[String]                                        = observable.tags
 }
 
 @DefineIndex(IndexType.standard, "data")
 @BuildVertexEntity
-case class Data(data: String)
+case class Data(data: String, fullData: Option[String])
+
+object UseHashToIndex extends ImmenseTermProcessor with ImmenseStringTermFilter {
+  override val termSizeLimit: Int = 8191
+  private val hasher: Hasher      = Hasher("SHA-256")
+
+  def hashToIndex(value: String): Option[String] =
+    if (value.length > termSizeLimit) Some("sha256/" + hasher.fromString(value).head.toString)
+    else None
+
+  override def apply[V](vertex: Vertex, property: VertexProperty[V]): Boolean = {
+    if (property.key() == "data")
+      vertex.label() match {
+        case "Observable" =>
+          collect(vertex, property).foreach { strProp =>
+            val currentValue = strProp.value()
+            logger.info(s"""Use hash for observable ~${vertex.id()}:
+                           |  dataType=${UMapping.string.getProperty(vertex, "dataType")}
+                           |  data=$currentValue
+                           |  message=${UMapping.string.optional.getProperty(vertex, "message").getOrElse("<not set>")}
+                           |  tags=${UMapping.string.sequence.getProperty(vertex, "message").mkString(", ")}""".stripMargin)
+            strProp.remove()
+            vertex.property(strProp.key(), "sha256/" + hasher.fromString(currentValue).head.toString)
+          }
+
+        case "Data" =>
+          collect(vertex, property).foreach { strProp =>
+            val currentValue = strProp.value()
+            logger.info(s"Use hash and move data for $vertex/${strProp.key()}: $currentValue")
+            strProp.remove()
+            vertex.property(strProp.key(), hasher.fromString(currentValue).head.toString)
+            vertex.property("fullData", currentValue)
+          }
+
+        case _ =>
+      }
+    false
+  }
+}