Merge branch 'hotfix/4.1.20'

To-om · To-om · commit 906d8bca7e63 · 2022-05-16T13:36:17.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Change Log
 
+## [4.1.20](https://github.com/TheHive-Project/TheHive/milestone/90) (2022-05-16)
+
+**Implemented enhancements:**
+
+- [Enhancement] Improve artifact creation from responder operation [\#2383](https://github.com/TheHive-Project/TheHive/issues/2383)
+- [Enhancement] Accept operations in analyzer reports [\#2384](https://github.com/TheHive-Project/TheHive/issues/2384)
+- [Enhancement] Add queries to filter observables based on the type of object it belongs to [\#2385](https://github.com/TheHive-Project/TheHive/issues/2385)
+
+**Fixed bugs:**
+
+- [Bug] Unable to use AWS S3 as storage backend  [\#2316](https://github.com/TheHive-Project/TheHive/issues/2316)
+- [Bug] Typo on migration elasticsearch http config [\#2374](https://github.com/TheHive-Project/TheHive/issues/2374)
+
 ## [4.1.19](https://github.com/TheHive-Project/TheHive/milestone/89) (2022-04-07)
 
 **Implemented enhancements:**
diff --git a/ScalliGraph b/ScalliGraph
@@ -1 +1 @@
-Subproject commit 6a590119ff69a3684e5f9dcc49a3f6e6e848c37f
+Subproject commit bdf6b0beb75643fa4300b2d1391aa8d7b3da85b8
diff --git a/build.sbt b/build.sbt
@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.19-1"
+val thehiveVersion         = "4.1.20-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/Conversion.scala
@@ -47,6 +47,7 @@ object Conversion {
       .withFieldComputed(_.id, _._id.toString)
       .withFieldConst(_._type, "case_artifact_job")
       .withFieldConst(_.case_artifact, None)
+      .withFieldComputed(_.operations, a => JsArray(a.operations).toString)
       .enableMethodAccessors
       .transform
   )
@@ -80,6 +81,7 @@ object Conversion {
               Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(Left(richCase)))))
           }
         )
+        .withFieldComputed(_.operations, a => JsArray(a.operations).toString)
         .enableMethodAccessors
         .transform
     }
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/ActionOperation.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/ActionOperation.scala
@@ -7,16 +7,25 @@ import play.api.libs.json._
   */
 trait ActionOperation
 
-case class AddTagToCase(tag: String)                                          extends ActionOperation
-case class AddTagToArtifact(tag: String)                                      extends ActionOperation
-case class CreateTask(title: String, description: String)                     extends ActionOperation
-case class AddCustomFields(name: String, tpe: String, value: JsValue)         extends ActionOperation
-case class CloseTask()                                                        extends ActionOperation
-case class MarkAlertAsRead()                                                  extends ActionOperation
-case class AddLogToTask(content: String, owner: Option[String])               extends ActionOperation
-case class AddTagToAlert(tag: String)                                         extends ActionOperation
-case class AddArtifactToCase(data: String, dataType: String, message: String) extends ActionOperation
-case class AssignCase(owner: String)                                          extends ActionOperation
+case class AddTagToCase(tag: String)                                  extends ActionOperation
+case class AddTagToArtifact(tag: String)                              extends ActionOperation
+case class CreateTask(title: String, description: String)             extends ActionOperation
+case class AddCustomFields(name: String, tpe: String, value: JsValue) extends ActionOperation
+case class CloseTask()                                                extends ActionOperation
+case class MarkAlertAsRead()                                          extends ActionOperation
+case class AddLogToTask(content: String, owner: Option[String])       extends ActionOperation
+case class AddTagToAlert(tag: String)                                 extends ActionOperation
+case class AddArtifactToCase(
+    data: String,
+    dataType: String,
+    message: String,
+    tlp: Option[Int],
+    ioc: Option[Boolean],
+    sighted: Option[Boolean],
+    ignoreSimilarity: Option[Boolean],
+    tags: Option[Seq[String]]
+)                                    extends ActionOperation
+case class AssignCase(owner: String) extends ActionOperation
 
 object ActionOperation {
   val addTagToCaseFormat: OFormat[AddTagToCase]           = Json.format[AddTagToCase]
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala
@@ -29,7 +29,8 @@ case class Job(
     endDate: Date, // end date of the job or if it is not finished date of the last check
     report: Option[JsObject],
     cortexId: String,
-    cortexJobId: String
+    cortexJobId: String,
+    operations: Seq[JsObject]
 )
 
 case class RichJob(
@@ -50,5 +51,5 @@ case class RichJob(
   def report: Option[JsObject]   = job.report
   def cortexId: String           = job.cortexId
   def cortexJobId: String        = job.cortexJobId
-
+  def operations: Seq[JsObject]  = job.operations
 }
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala
@@ -93,20 +93,20 @@ class ActionOperationSrv @Inject() (
           _ <- logSrv.create(Log(content, new Date()), t, None)
         } yield updateOperation(operation)
 
-      case AddArtifactToCase(data, dataType, message) =>
+      case AddArtifactToCase(data, dataType, message, tlp, ioc, sighted, ignoreSimilarity, tags) =>
         for {
           c            <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action AddArtifactToCase without case")))(Success(_))
           organisation <- organisationSrv.getOrFail(authContext.organisation)
           _ <- caseSrv.createObservable(
             c,
             Observable(
               message = Some(message),
-              tlp = 2,
-              ioc = false,
-              sighted = false,
-              ignoreSimilarity = None,
+              tlp = tlp.getOrElse(2),
+              ioc = ioc.getOrElse(false),
+              sighted = sighted.getOrElse(false),
+              ignoreSimilarity = ignoreSimilarity,
               dataType = dataType,
-              tags = Nil,
+              tags = tags.getOrElse(Nil),
               relatedId = c._id,
               organisationIds = Set(organisation._id)
             ),
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala
@@ -96,7 +96,7 @@ class ActionSrv @Inject() (
         job.report.flatMap(_.full),
         client.name,
         job.id,
-        job.report.fold[Seq[JsObject]](Nil)(_.operations)
+        Nil
       )
       createdAction <- Future.fromTry {
         db.tryTransaction { implicit graph =>
diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala
@@ -42,6 +42,7 @@ class JobSrv @Inject() (
     observableTypeSrv: ObservableTypeSrv,
     attachmentSrv: AttachmentSrv,
     reportTagSrv: ReportTagSrv,
+    actionOperationSrv: ActionOperationSrv,
     serviceHelper: ServiceHelper,
     auditSrv: CortexAuditSrv,
     organisationSrv: OrganisationSrv,
@@ -130,6 +131,7 @@ class JobSrv @Inject() (
       .withFieldConst(_.report, None)
       .withFieldConst(_.cortexId, "tbd")
       .withFieldComputed(_.cortexJobId, _.id)
+      .withFieldConst(_.operations, Nil)
       .transform
 
   /**
@@ -167,11 +169,31 @@ class JobSrv @Inject() (
           .availableCortexClients(connector.clients, authContext.organisation)
           .find(_.name == cortexId)
           .fold[Future[CortexClient]](Future.failed(NotFoundError(s"Cortex $cortexId not found")))(Future.successful)
-      job <- Future.fromTry(updateJobStatus(jobId, cortexJob))
-      _   <- importCortexArtifacts(job, cortexJob, cortexClient)
-      _   <- Future.fromTry(importAnalyzerTags(job, cortexJob))
+      operations <- Future.fromTry(executeOperations(jobId, cortexJob))
+      job        <- Future.fromTry(updateJobStatus(jobId, cortexJob, operations))
+      _          <- importCortexArtifacts(job, cortexJob, cortexClient)
+      _          <- Future.fromTry(importAnalyzerTags(job, cortexJob))
     } yield job
 
+  def executeOperations(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Seq[ActionOperationStatus]] =
+    db.tryTransaction { implicit graph =>
+      get(jobId)
+        .observable
+        .project(_.by.by(_.`case`.option))
+        .getOrFail("Observable")
+        .map {
+          case (relatedObservable, relatedCase) =>
+            cortexJob
+              .report
+              .fold[Seq[ActionOperation]](Nil)(_.operations.map(_.as[ActionOperation]))
+              .map { operation =>
+                actionOperationSrv
+                  .execute(relatedObservable, operation, relatedCase, None)
+                  .fold(t => ActionOperationStatus(operation, success = false, t.getMessage), identity)
+              }
+        }
+    }
+
   /**
     * Update job status, set the endDate and remove artifacts from report
     *
@@ -180,7 +202,9 @@ class JobSrv @Inject() (
     * @param authContext the authentication context
     * @return the updated job
     */
-  private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob)(implicit authContext: AuthContext): Try[Job with Entity] =
+  private def updateJobStatus(jobId: EntityId, cortexJob: CortexJob, operations: Seq[ActionOperationStatus])(implicit
+      authContext: AuthContext
+  ): Try[Job with Entity] =
     db.tryTransaction { implicit graph =>
       getOrFail(jobId).flatMap { job =>
         val report  = cortexJob.report.flatMap(r => r.full orElse r.errorMessage.map(m => Json.obj("errorMessage" -> m)))
@@ -193,6 +217,7 @@ class JobSrv @Inject() (
             .update(_.endDate, endDate)
             .update(_._updatedAt, Some(new Date))
             .update(_._updatedBy, Some(authContext.userId))
+            .update(_.operations, operations.map(o => Json.toJsObject(o)))
             .getOrFail("Job")
           observable <- get(job).observable.getOrFail("Observable")
           _ <-
diff --git a/cortex/connector/src/test/resources/cortex-jobs.json b/cortex/connector/src/test/resources/cortex-jobs.json
@@ -87,12 +87,21 @@
         {
           "data": "192.168.1.1",
           "message": "myIp",
-          "tags": [],
+          "tags": ["tag-test"],
           "tlp": 2,
           "dataType": "ip"
         }
       ],
       "operations": [
+        {
+          "type": "AddArtifactToCase",
+          "data": "myData",
+          "dataType": "other",
+          "message": "test-operation",
+          "tlp": 3,
+          "ignoreSimilarity": false,
+          "tags": ["tag1", "tag2"]
+        }
       ]
     },
     "tlp": 2,
diff --git a/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala b/cortex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/JobSrvTest.scala
@@ -7,9 +7,10 @@ import org.thp.scalligraph.models.{Database, DummyUserSrv, Schema}
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{AppBuilder, EntityName}
 import org.thp.thehive.TestAppBuilder
-import org.thp.thehive.connector.cortex.models.{Job, JobStatus, TheHiveCortexSchemaProvider}
+import org.thp.thehive.connector.cortex.models.{ActionOperationStatus, Job, JobStatus, TheHiveCortexSchemaProvider}
 import org.thp.thehive.connector.cortex.services.JobOps._
 import org.thp.thehive.models.Permissions
+import org.thp.thehive.services.CaseOps._
 import org.thp.thehive.services.ObservableOps._
 import org.thp.thehive.services.UserOps._
 import org.thp.thehive.services._
@@ -23,7 +24,8 @@ import scala.concurrent.duration.DurationInt
 import scala.io.Source
 
 class JobSrvTest extends PlaySpecification with TestAppBuilder {
-  implicit val authContext: AuthContext = DummyUserSrv(userId = "admin@thehive.local", permissions = Permissions.all).authContext
+  implicit val authContext: AuthContext =
+    DummyUserSrv(userId = "certuser@thehive.local", organisation = "cert", permissions = Permissions.all).authContext
   override def appConfigure: AppBuilder =
     super
       .appConfigure
@@ -33,53 +35,76 @@ class JobSrvTest extends PlaySpecification with TestAppBuilder {
       .`override`(_.bindToProvider[Schema, TheHiveCortexSchemaProvider])
 
   "job service" should {
+    val cortexOutputJob = {
+      val dataSource = Source.fromResource("cortex-jobs.json")
+      val data       = dataSource.mkString
+      dataSource.close()
+      Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get
+    }
+
     "handle creation and then finished job" in testApp { app =>
-//      val job = Job(
-//        workerId = "anaTest2",
-//        workerName = "anaTest2",
-//        workerDefinition = "test2",
-//        status = JobStatus.Waiting,
-//        startDate = new Date(1561625908856L),
-//        endDate = new Date(1561625908856L),
-//        report = None,
-//        cortexId = "test",
-//        cortexJobId = "LVyYKFstq3Rtrdc9DFmL"
-//      )
-//
-//      val cortexOutputJob = {
-//        val dataSource = Source.fromResource("cortex-jobs.json")
-//        val data       = dataSource.mkString
-//        dataSource.close()
-//        Json.parse(data).as[List[OutputJob]].find(_.id == "ZWu85Q1OCVNx03hXK4df").get
-//      }
-//
-//      val createdJobTry = app[Database].tryTransaction { implicit graph =>
-//        for {
-//          observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable")
-//          createdJob <- app[JobSrv].create(job, observable)
-//        } yield createdJob
-//      }
-//      createdJobTry.map { createdJob =>
-//        Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds)
-//      } must beASuccessfulTry.which { updatedJob =>
-//        updatedJob.status shouldEqual JobStatus.Success
-//        updatedJob.report must beSome
-//        (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg"
-//
-//        app[Database].roTransaction { implicit graph =>
-//          app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue
-//          app[JobSrv].get(updatedJob).reportObservables.toList.length must equalTo(2).updateMessage { s =>
-//            s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}"
-//          }
-//
-//          for {
-//            audit        <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit")
-//            organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation")
-//            user         <- app[UserSrv].startTraversal.getByName("certuser@thehive.local").getOrFail("User")
-//          } yield new JobFinished().filter(audit, Some(updatedJob), organisation, Some(user))
-//        } must beASuccessfulTry(true)
-//      }
-      pending("flaky test")
+      val job = Job(
+        workerId = "anaTest2",
+        workerName = "anaTest2",
+        workerDefinition = "test2",
+        status = JobStatus.Waiting,
+        startDate = new Date(1561625908856L),
+        endDate = new Date(1561625908856L),
+        report = None,
+        cortexId = "test",
+        cortexJobId = "LVyYKFstq3Rtrdc9DFmL",
+        operations = Nil
+      )
+
+      val createdJobTry = app[Database].tryTransaction { implicit graph =>
+        for {
+          observable <- app[ObservableSrv].startTraversal.has(_.message, "hello world").getOrFail("Observable")
+          createdJob <- app[JobSrv].create(job, observable)
+        } yield createdJob
+      }
+      val finishedJobTry = createdJobTry.map { createdJob =>
+        Await.result(app[JobSrv].finished(app[CortexClient].name, createdJob._id, cortexOutputJob), 20.seconds)
+      }
+      finishedJobTry must beASuccessfulTry
+      val updatedJob = finishedJobTry.get
+      updatedJob.status shouldEqual JobStatus.Success
+      updatedJob.report must beSome
+      (updatedJob.report.get \ "data").as[String] shouldEqual "imageedit_2_3904987689.jpg"
+      updatedJob.operations must haveSize(1)
+      updatedJob.operations.map(o => (o \ "status").as[String]) must contain(beEqualTo("Success"))
+        .forall
+        .updateMessage(s => s"$s\nOperation has failed: ${updatedJob.operations.map("\n -" + _).mkString}")
+
+      app[Database].roTransaction { implicit graph =>
+        app[JobSrv].get(updatedJob).observable.has(_.message, "hello world").exists must beTrue
+        val reportObservables = app[JobSrv].get(updatedJob).reportObservables.toSeq
+        reportObservables.length must equalTo(2).updateMessage { s =>
+          s"$s\nreport observables are : ${app[JobSrv].get(updatedJob).reportObservables.richObservable.toList.mkString("\n")}"
+        }
+        val ipObservable = reportObservables.find(_.dataType == "ip").get
+        ipObservable.data    must beSome("192.168.1.1")
+        ipObservable.message must beSome("myIp")
+        ipObservable.tags    must contain(exactly("tag-test"))
+        ipObservable.tlp     must beEqualTo(2)
+
+        val operationObservableMaybe = app[JobSrv]
+          .get(updatedJob)
+          .observable
+          .`case`
+          .observables
+          .has(_.message, "test-operation")
+          .headOption
+        operationObservableMaybe must beSome.which { operationObservable =>
+          operationObservable.data must beSome("myData")
+          operationObservable.tlp  must beEqualTo(3)
+          operationObservable.tags must contain(exactly("tag1", "tag2"))
+        }
+        for {
+          audit        <- app[AuditSrv].startTraversal.has(_.objectId, updatedJob._id.toString).getOrFail("Audit")
+          organisation <- app[OrganisationSrv].getByName("cert").getOrFail("Organisation")
+          user         <- app[UserSrv].startTraversal.getByName("certadmin@thehive.local").getOrFail("User")
+        } yield JobFinished.filter(audit, Some(updatedJob), organisation, Some(user))
+      } must beASuccessfulTry(true).setMessage("The audit doesn't match the expected criteria")
     }
 
     "submit a job" in testApp { app =>
diff --git a/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala b/dto/src/main/scala/org/thp/thehive/connector/cortex/dto/v0/Job.scala
@@ -17,7 +17,8 @@ case class OutputJob(
     cortexId: String,
     cortexJobId: String,
     id: String,
-    case_artifact: Option[OutputObservable]
+    case_artifact: Option[OutputObservable],
+    operations: String
 )
 
 object OutputJob {
diff --git a/frontend/bower.json b/frontend/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.1.19-1",
+  "version": "4.1.20-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",
diff --git a/frontend/package.json b/frontend/package.json
@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.1.19-1",
+    "version": "4.1.20-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",
diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/Conversion.scala
diff --git a/migration/src/main/scala/org/thp/thehive/migration/th3/ElasticClient.scala b/migration/src/main/scala/org/thp/thehive/migration/th3/ElasticClient.scala
diff --git a/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v0/ObservableCtrl.scala
diff --git a/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala b/thehive/app/org/thp/thehive/controllers/v1/ObservableCtrl.scala

Original file line number	Diff line number	Diff line change
`@@ -47,6 +47,7 @@ object Conversion {`
`47`	`47`	`.withFieldComputed(_.id, _._id.toString)`
`48`	`48`	`.withFieldConst(_._type, "case_artifact_job")`
`49`	`49`	`.withFieldConst(_.case_artifact, None)`
	`50`	`+ .withFieldComputed(_.operations, a => JsArray(a.operations).toString)`
`50`	`51`	`.enableMethodAccessors`
`51`	`52`	`.transform`
`52`	`53`	`)`
`@@ -80,6 +81,7 @@ object Conversion {`
`80`	`81`	`Some(observableWithExtraOutput.toValue((richObservable, JsObject.empty, Some(Left(richCase)))))`
`81`	`82`	`}`
`82`	`83`	`)`
	`84`	`+ .withFieldComputed(_.operations, a => JsArray(a.operations).toString)`
`83`	`85`	`.enableMethodAccessors`
`84`	`86`	`.transform`
`85`	`87`	`}`
Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ class ActionSrv @Inject() (`
`96`	`96`	`job.report.flatMap(_.full),`
`97`	`97`	`client.name,`
`98`	`98`	`job.id,`
`99`		`- job.report.fold[Seq[JsObject]](Nil)(_.operations)`
	`99`	`+ Nil`
`100`	`100`	`)`
`101`	`101`	`createdAction <- Future.fromTry {`
`102`	`102`	`db.tryTransaction { implicit graph =>`