Merge branch 'hotfix/4.1.17'

Author: To-om

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Change Log
 
+## [4.1.17](https://github.com/TheHive-Project/TheHive/milestone/87) (2022-01-24)
+
+**Implemented enhancements:**
+
+- [Enhancement] Improve migration tool by accepting old versions of TheHive [\#2305](https://github.com/TheHive-Project/TheHive/issues/2305)
+- Security concern [\#2309](https://github.com/TheHive-Project/TheHive/issues/2309)
+
+**Fixed bugs:**
+
+- [Bug] Action 'mergeCase' not mapped in v0  [\#2304](https://github.com/TheHive-Project/TheHive/issues/2304)
+- Can't start after upgrade thehive4 (4.1.16-1) over (4.0.0-1) [Bug] [\#2308](https://github.com/TheHive-Project/TheHive/issues/2308)
+- [Bug] Notifications are executed several times [\#2317](https://github.com/TheHive-Project/TheHive/issues/2317)
+
 ## [4.1.16](https://github.com/TheHive-Project/TheHive/milestone/86) (2021-12-17)
 
 **Implemented enhancements:**

ScalliGraph

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.16-1"
+val thehiveVersion         = "4.1.17-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -342,10 +342,7 @@ lazy val thehiveMigration = (project in file("migration"))
     resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven",
     crossScalaVersions := Seq(scala212),
     libraryDependencies ++= Seq(
-      elastic4sCore,
-      elastic4sHttpStreams,
-      elastic4sClient,
-//      jts,
+      alpakka,
       ehcache,
       scopt,
       specs % Test

build.sbt

@@ -44,6 +44,7 @@
     <logger name="org.thp.thehive.services.LocalAuthSrv" level="TRACE" />
     <logger name="org.thp.scalligraph.graphql" level="TRACE" />
     -->
+    <logger name="org.thp.scalligraph.traversal" level="INFO"/>
     <logger name="org.janusgraph.graphdb.transaction.StandardJanusGraphTx" level="ERROR"/>
     <logger name="org.thp.thehive" level="INFO"/>
 

conf/migration-logback.xml

@@ -1,7 +1,7 @@
 package org.thp.thehive.dto.v1
 
 import org.thp.scalligraph.controllers.FFile
-import play.api.libs.json.{Json, OFormat, Writes}
+import play.api.libs.json.{JsObject, Json, OFormat, Writes}
 
 import java.util.Date
 
@@ -32,7 +32,8 @@ case class OutputUser(
     permissions: Set[String],
     organisation: String,
     avatar: Option[String],
-    organisations: Seq[OutputOrganisationProfile]
+    organisations: Seq[OutputOrganisationProfile],
+    extraData: JsObject
 )
 
 object OutputUser {

dto/src/main/scala/org/thp/thehive/dto/v1/User.scala

@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.16-1",
+  "version": "4.1.17-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

frontend/bower.json

@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.16-1",
+    "version": "4.1.17-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

frontend/package.json

@@ -10,11 +10,13 @@ input {
     keepalive: 10h
     # Size of the page for scroll
     pagesize: 10
+
+    maxAttempts = 5
+    minBackoff = 10 milliseconds
+    maxBackoff = 10 seconds
+    randomFactor = 0.2
   }
   filter {
-    maxCaseAge: 0
-    maxAlertAge: 0
-    maxAuditAge: 0
     includeAlertTypes: []
     excludeAlertTypes: []
     includeAlertSources: []
@@ -39,6 +41,7 @@ input {
 
 output {
   caseNumberShift: 0
+  resume: false
   removeData: false
   db {
     provider: janusgraph
@@ -77,6 +80,8 @@ output {
   }
 }
 
+threadCount: 4
+transactionPageSize: 50
 
 from {
   db {

migration/src/main/resources/reference.conf

@@ -44,21 +44,21 @@ trait IntegrityCheckApp {
               bind[ActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider]
 
               val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder)
-              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
               integrityCheckOpsBindings.addBinding.to[CaseTemplateIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[CustomFieldIntegrityCheckOps]
               integrityCheckOpsBindings.addBinding.to[DataIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[CaseIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[AlertIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
-              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[ImpactStatusIntegrityCheckOps]
               integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[ObservableTypeIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[OrganisationIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[ResolutionStatusIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[TagIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[TaskIntegrityCheckOps]
+              integrityCheckOpsBindings.addBinding.to[UserIntegrityCheckOps]
 
               bind[Environment].toInstance(Environment.simple())
               bind[ApplicationLifecycle].to[DefaultApplicationLifecycle]

migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala

@@ -48,17 +48,15 @@ object Filter {
               new ParseException(s"Unparseable date: $s\nExpected format is ${dateFormats.map(_.toPattern).mkString("\"", "\" or \"", "\"")}", 0)
             )
         }
-    def readDate(dateConfigName: String, ageConfigName: String) =
+    def readDate(dateConfigName: String, ageConfigName: String): Option[Long] =
       Try(config.getString(dateConfigName))
         .flatMap(parseDate)
         .map(d => d.getTime)
-        .toOption
         .orElse {
-          Try {
-            val age = config.getDuration(ageConfigName)
-            if (age.isZero) None else Some(now - age.getSeconds * 1000)
-          }.toOption.flatten
+          Try(config.getDuration(ageConfigName))
+            .map(d => now - d.getSeconds * 1000)
         }
+        .toOption
     val caseFromDate    = readDate("caseFromDate", "maxCaseAge")
     val caseUntilDate   = readDate("caseUntilDate", "minCaseAge")
     val caseFromNumber  = Try(config.getInt("caseFromNumber")).toOption
@@ -90,24 +88,16 @@ trait Input {
   def countOrganisations(filter: Filter): Future[Long]
   def listCases(filter: Filter): Source[Try[InputCase], NotUsed]
   def countCases(filter: Filter): Future[Long]
-  def listCaseObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
   def countCaseObservables(filter: Filter): Future[Long]
   def listCaseObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
-  def countCaseObservables(caseId: String): Future[Long]
-  def listCaseTasks(filter: Filter): Source[Try[(String, InputTask)], NotUsed]
   def countCaseTasks(filter: Filter): Future[Long]
   def listCaseTasks(caseId: String): Source[Try[(String, InputTask)], NotUsed]
-  def countCaseTasks(caseId: String): Future[Long]
-  def listCaseTaskLogs(filter: Filter): Source[Try[(String, InputLog)], NotUsed]
   def countCaseTaskLogs(filter: Filter): Future[Long]
   def listCaseTaskLogs(caseId: String): Source[Try[(String, InputLog)], NotUsed]
-  def countCaseTaskLogs(caseId: String): Future[Long]
   def listAlerts(filter: Filter): Source[Try[InputAlert], NotUsed]
   def countAlerts(filter: Filter): Future[Long]
-  def listAlertObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
   def countAlertObservables(filter: Filter): Future[Long]
   def listAlertObservables(alertId: String): Source[Try[(String, InputObservable)], NotUsed]
-  def countAlertObservables(alertId: String): Future[Long]
   def listUsers(filter: Filter): Source[Try[InputUser], NotUsed]
   def countUsers(filter: Filter): Future[Long]
   def listCustomFields(filter: Filter): Source[Try[InputCustomField], NotUsed]
@@ -123,25 +113,13 @@ trait Input {
   def listCaseTemplate(filter: Filter): Source[Try[InputCaseTemplate], NotUsed]
   def countCaseTemplate(filter: Filter): Future[Long]
   def listCaseTemplateTask(caseTemplateId: String): Source[Try[(String, InputTask)], NotUsed]
-  def countCaseTemplateTask(caseTemplateId: String): Future[Long]
-  def listCaseTemplateTask(filter: Filter): Source[Try[(String, InputTask)], NotUsed]
   def countCaseTemplateTask(filter: Filter): Future[Long]
   def listJobs(caseId: String): Source[Try[(String, InputJob)], NotUsed]
-  def countJobs(caseId: String): Future[Long]
-  def listJobs(filter: Filter): Source[Try[(String, InputJob)], NotUsed]
   def countJobs(filter: Filter): Future[Long]
-  def listJobObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed]
   def countJobObservables(filter: Filter): Future[Long]
   def listJobObservables(caseId: String): Source[Try[(String, InputObservable)], NotUsed]
-  def countJobObservables(caseId: String): Future[Long]
-  def listAction(filter: Filter): Source[Try[(String, InputAction)], NotUsed]
   def countAction(filter: Filter): Future[Long]
-  def listAction(entityId: String): Source[Try[(String, InputAction)], NotUsed]
   def listActions(entityIds: Seq[String]): Source[Try[(String, InputAction)], NotUsed]
-  def countAction(entityId: String): Future[Long]
-  def listAudit(filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
   def countAudit(filter: Filter): Future[Long]
-  def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
   def listAudits(entityIds: Seq[String], filter: Filter): Source[Try[(String, InputAudit)], NotUsed]
-  def countAudit(entityId: String, filter: Filter): Future[Long]
 }

migration/src/main/scala/org/thp/thehive/migration/Input.scala

@@ -10,13 +10,16 @@ import scopt.OParser
 import java.io.File
 import java.nio.file.{Files, Paths}
 import scala.collection.JavaConverters._
-import scala.concurrent.duration.{Duration, DurationInt}
-import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.{blocking, Await, ExecutionContext, Future}
 
 object Migrate extends App with MigrationOps {
   val defaultLoggerConfigFile = "/etc/thehive/logback-migration.xml"
   if (System.getProperty("logger.file") == null && Files.exists(Paths.get(defaultLoggerConfigFile)))
     System.setProperty("logger.file", defaultLoggerConfigFile)
+  (new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)
+  var transactionPageSize: Int = 100
+  var threadCount: Int         = 3
 
   def getVersion: String = Option(getClass.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
 
@@ -53,6 +56,9 @@ object Migrate extends App with MigrationOps {
       opt[Unit]('d', "drop-database")
         .action((_, c) => addConfig(c, "output.dropDatabase", true))
         .text("Drop TheHive4 database before migration"),
+      opt[Unit]('r', "resume")
+        .action((_, c) => addConfig(c, "output.resume", true))
+        .text("Resume migration (or migrate on existing database)"),
       opt[String]('m', "main-organisation")
         .valueName("<organisation>")
         .action((o, c) => addConfig(c, "input.mainOrganisation", o)),
@@ -64,13 +70,27 @@ object Migrate extends App with MigrationOps {
         .valueName("<index>")
         .text("TheHive3 ElasticSearch index name")
         .action((i, c) => addConfig(c, "input.search.index", i)),
+      opt[String]('x', "es-index-version")
+        .valueName("<index>")
+        .text("TheHive3 ElasticSearch index name version number (default: autodetect)")
+        .action((i, c) => addConfig(c, "input.search.indexVersion", i)),
       opt[String]('a', "es-keepalive")
         .valueName("<duration>")
         .text("TheHive3 ElasticSearch keepalive")
         .action((a, c) => addConfig(c, "input.search.keepalive", a)),
       opt[Int]('p', "es-pagesize")
         .text("TheHive3 ElasticSearch page size")
         .action((p, c) => addConfig(c, "input.search.pagesize", p)),
+      opt[Boolean]('s', "es-single-type")
+        .valueName("<bool>")
+        .text("Elasticsearch single type")
+        .action((s, c) => addConfig(c, "input.search.singleType", s)),
+      opt[Int]('y', "transaction-pagesize")
+        .text("page size for each transaction")
+        .action((t, c) => addConfig(c, "transactionPageSize", t)),
+      opt[Int]('t', "thread-count")
+        .text("number of threads")
+        .action((t, c) => addConfig(c, "threadCount", t)),
       /* case age */
       opt[String]("max-case-age")
         .valueName("<duration>")
@@ -134,11 +154,11 @@ object Migrate extends App with MigrationOps {
       opt[String]("max-audit-age")
         .valueName("<duration>")
         .text("migrate only audits whose age is less than <duration>")
-        .action((v, c) => addConfig(c, "input.filter.minAuditAge", v)),
+        .action((v, c) => addConfig(c, "input.filter.maxAuditAge", v)),
       opt[String]("min-audit-age")
         .valueName("<duration>")
         .text("migrate only audits whose age is greater than <duration>")
-        .action((v, c) => addConfig(c, "input.filter.maxAuditAge", v)),
+        .action((v, c) => addConfig(c, "input.filter.minAuditAge", v)),
       opt[String]("audit-from-date")
         .valueName("<date>")
         .text("migrate only audits created from <date>")
@@ -183,13 +203,19 @@ object Migrate extends App with MigrationOps {
     implicit val actorSystem: ActorSystem = ActorSystem("TheHiveMigration", config)
     implicit val ec: ExecutionContext     = actorSystem.dispatcher
     implicit val mat: Materializer        = Materializer(actorSystem)
+    transactionPageSize = config.getInt("transactionPageSize")
+    threadCount = config.getInt("threadCount")
+    var stop = false
 
     try {
-      (new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)
-
-      val timer = actorSystem.scheduler.scheduleAtFixedRate(10.seconds, 10.seconds) { () =>
-        logger.info(migrationStats.showStats())
-        migrationStats.flush()
+      Future {
+        blocking {
+          while (!stop) {
+            logger.info(migrationStats.showStats())
+            migrationStats.flush()
+            Thread.sleep(10000) // 10 seconds
+          }
+        }
       }
 
       val returnStatus =
@@ -198,17 +224,15 @@ object Migrate extends App with MigrationOps {
           val output = th4.Output(Configuration(config.getConfig("output").withFallback(config)))
           val filter = Filter.fromConfig(config.getConfig("input.filter"))
 
-          val process = migrate(input, output, filter)
-
-          Await.result(process, Duration.Inf)
+          migrate(input, output, filter).get
           logger.info("Migration finished")
           0
         } catch {
           case e: Throwable =>
             logger.error(s"Migration failed", e)
             1
         } finally {
-          timer.cancel()
+          stop = true
           Await.ready(actorSystem.terminate(), 1.minute)
           ()
         }