Merge branch 'hotfix/4.1.18'

Author: To-om

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Change Log
 
+## [4.1.18](https://github.com/TheHive-Project/TheHive/milestone/88) (2022-02-07)
+
+**Implemented enhancements:**
+
+- [Enhancement] Integrity check improvement [\#2334](https://github.com/TheHive-Project/TheHive/issues/2334)
+- [Enhancement] Improve migration tool [\#2335](https://github.com/TheHive-Project/TheHive/issues/2335)
+
+**Fixed bugs:**
+
+- [Bug] "Character 8211 cannot match AsciiSet because it is out of range" error when downloading a report [\#1534](https://github.com/TheHive-Project/TheHive/issues/1534)
+- [Bug] Can add a "space" as observable [\#2324](https://github.com/TheHive-Project/TheHive/issues/2324)
+- [Bug]- Migration from Hive 3.4.4 to Hive 4.1.17 not working [\#2331](https://github.com/TheHive-Project/TheHive/issues/2331)
+- [Bug] Duplicated entities after "db.janusgraph.forceDropAndRebuildIndex: true" with Elasticsearch index [\#2333](https://github.com/TheHive-Project/TheHive/issues/2333)
+- [Bug] Query with parendId filter doesn't work (v0) [\#2336](https://github.com/TheHive-Project/TheHive/issues/2336)
+
 ## [4.1.17](https://github.com/TheHive-Project/TheHive/milestone/87) (2022-01-24)
 
 **Implemented enhancements:**

ScalliGraph

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.17-1"
+val thehiveVersion         = "4.1.18-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -165,7 +165,8 @@ lazy val thehiveCore = (project in file("thehive"))
       pbkdf2,
       commonCodec,
       scalaGuice,
-      reflections
+      reflections,
+      quartzScheduler
     )
   )
 

build.sbt

@@ -50,7 +50,7 @@ class CortexQueryExecutor @Inject() (
 
   override val customFilterQuery: FilterQuery = FilterQuery(publicProperties) { (tpe, globalParser) =>
     FieldsParser("parentChildFilter") {
-      case (_, FObjOne("_parent", ParentIdFilter(_, parentId))) if parentTypes.isDefinedAt(tpe) =>
+      case (_, FObjOne("_parent", ParentIdFilter(parentId, _))) if parentTypes.isDefinedAt(tpe) =>
         Good(new CortexParentIdInputFilter(parentId))
       case (path, FObjOne("_parent", ParentQueryFilter(_, parentFilterField))) if parentTypes.isDefinedAt(tpe) =>
         globalParser(parentTypes(tpe)).apply(path, parentFilterField).map(query => new CortexParentQueryInputFilter(query))

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.17-1",
+  "version": "4.1.18-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/bower.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.17-1",
+    "version": "4.1.18-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

frontend/package.json

@@ -43,6 +43,7 @@ output {
   caseNumberShift: 0
   resume: false
   removeData: false
+  integrityCheck.enabled: false
   db {
     provider: janusgraph
     janusgraph {

migration/src/main/resources/reference.conf

@@ -20,6 +20,7 @@ import javax.inject.Inject
 import scala.collection.JavaConverters._
 import scala.collection.immutable
 import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.DurationInt
 import scala.util.Success
 
 trait IntegrityCheckApp {
@@ -43,22 +44,22 @@ trait IntegrityCheckApp {
               bindActor[DummyActor]("integrity-check-actor")
               bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
-              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
+              val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[IntegrityCheck](binder)
+              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[DataIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[LogIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheck]
+              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheck]
 
               bind[Environment].toInstance(Environment.simple())
               bind[ApplicationLifecycle].to[DefaultApplicationLifecycle]
@@ -77,25 +78,29 @@ trait IntegrityCheckApp {
     buildApp(configuration, db).getInstance(classOf[IntegrityChecks]).runChecks()
 }
 
-class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[GenIntegrityCheckOps], userSrv: UserDB) extends MapMerger {
+class IntegrityChecks @Inject() (db: Database, checks: immutable.Set[IntegrityCheck], userSrv: UserDB) extends MapMerger {
   def runChecks(): Unit = {
     implicit val authContext: AuthContext = userSrv.getSystemAuthContext
     checks.foreach { c =>
-      db.tryTransaction { implicit graph =>
-        println(s"Running check on ${c.name} ...")
-        c.initialCheck()
-        val stats = c.duplicationCheck() <+> c.globalCheck()
-        val statsStr = stats
-          .collect {
-            case (k, v) if v != 0 => s"$k:$v"
-          }
-          .mkString(" ")
-        if (statsStr.isEmpty)
-          println("  no change needed")
-        else
-          println(s"  $statsStr")
-        Success(())
+      println(s"Running check on ${c.name} ...")
+      val desupStats = c match {
+        case dc: DedupCheck[_] => dc.dedup(KillSwitch.alwaysOn)
+        case _                 => Map.empty[String, Long]
       }
+      val globalStats = c match {
+        case gc: GlobalCheck[_] => gc.runGlobalCheck(24.hours, KillSwitch.alwaysOn)
+        case _                  => Map.empty[String, Long]
+      }
+      val statsStr = (desupStats <+> globalStats)
+        .collect {
+          case (k, v) if v != 0 => s"$k:$v"
+        }
+        .mkString(" ")
+      if (statsStr.isEmpty)
+        println("  no change needed")
+      else
+        println(s"  $statsStr")
+
     }
   }
 }

migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

@@ -120,6 +120,8 @@ trait Input {
   def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
   def countAction(filter: Filter): Future[Long]
   def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed]
-  def countAudit(filter: Filter): Future[Long]
+  def countAudits(filter: Filter): Future[Long]
   def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
+  def countDashboards(filter: Filter): Future[Long]
+  def listDashboards(filter: Filter): Source[Try[InputDashboard], NotUsed]
 }

migration/src/main/scala/org/thp/thehive/migration/Input.scala

@@ -91,6 +91,9 @@ object Migrate extends App with MigrationOps {
       opt[Int]('t', "thread-count")
         .text("number of threads")
         .action((t, c) => addConfig(c, "threadCount", t)),
+      opt[Unit]('k', "integrity-checks")
+        .text("run integrity checks after the migration")
+        .action((_, c) => addConfig(c, "output.integrityCheck.enabled", true)),
       /* case age */
       opt[String]("max-case-age")
         .valueName("<duration>")