JPERF-1106: Gather Apache2 logs

pczuj · pczuj · commit af09430d9493 · 2023-05-27T20:31:54.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,19 @@ Dropping a requirement of a major version of a dependency is a new contract.
 ## [Unreleased]
 [Unreleased]: https://github.com/atlassian/aws-infrastructure/compare/release-3.0.0...master
 
+## Added
+- Add `DiagnosableLoadBalancer`.
+- Add ability to `gatherEvidence` from `ApacheProxyLoadBalancer` by making it `DiagnosableLoadBalancer`.
+- Make it possible to specify `extraMeasurementSources` inside `Jira`, so that it's possible to `gatherResults` from any of Jira hosted integrations, e.g. load balancer or plugins.
+- Let `Jira` produced by `DataCenterFormula` gather logs of `ApacheProxyLoadBalancer`. Resolve [JPERF-1106].
+- Make `StartedNode` a `MeasurementSource`.
+
+## Fixed
+- Let every `MeasurementSource` inside `Jira` finish its gathering even if one of them fails. Fix [JPERF-1114].
+
+[JPERF-1106]: https://ecosystem.atlassian.net/browse/JPERF-1106
+[JPERF-1114]: https://ecosystem.atlassian.net/browse/JPERF-1114
+
 ## [3.0.0] - 2023-05-25
 [3.0.0]: https://github.com/atlassian/aws-infrastructure/compare/release-2.29.0...release-3.0.0
 
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt
@@ -0,0 +1,26 @@
+package com.atlassian.performance.tools.awsinfrastructure
+
+internal class FailSafeRunnable(
+    private val delegates: Iterable<Runnable>
+) : Runnable {
+    override fun run() {
+        val exceptions = delegates.mapNotNull {
+            try {
+                it.run()
+                null
+            } catch (e: Exception) {
+                e
+            }
+        }
+
+        when {
+            exceptions.isEmpty() -> return
+            exceptions.size == 1 -> throw exceptions[0]
+            else -> {
+                val root = Exception("Multiple exceptions were thrown and are added suppressed into this one")
+                exceptions.forEach { root.addSuppressed(it) }
+                throw root
+            }
+        }
+    }
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt
@@ -11,6 +11,7 @@ import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M4ExtraLar
 import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume
 import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheEc2LoadBalancerFormula
 import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheProxyLoadBalancer
+import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer
 import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.LoadBalancerFormula
 import com.atlassian.performance.tools.awsinfrastructure.api.network.Network
 import com.atlassian.performance.tools.awsinfrastructure.api.network.NetworkFormula
@@ -19,6 +20,7 @@ import com.atlassian.performance.tools.awsinfrastructure.jira.DataCenterNodeForm
 import com.atlassian.performance.tools.awsinfrastructure.jira.DiagnosableNodeFormula
 import com.atlassian.performance.tools.awsinfrastructure.jira.StandaloneNodeFormula
 import com.atlassian.performance.tools.awsinfrastructure.jira.home.SharedHomeFormula
+import com.atlassian.performance.tools.awsinfrastructure.loadbalancer.LoadBalancerMeasurementSource.Extension.asMeasurementSource
 import com.atlassian.performance.tools.concurrency.api.AbruptExecutorService
 import com.atlassian.performance.tools.concurrency.api.submitWithLogContext
 import com.atlassian.performance.tools.infrastructure.api.app.Apps
@@ -326,6 +328,8 @@ class DataCenterFormula private constructor(
             loadBalancer.waitUntilHealthy(Duration.ofMinutes(5))
         }
 
+        val loadBalancerResultsSource = (provisionedLoadBalancer.loadBalancer as? DiagnosableLoadBalancer)
+            ?.asMeasurementSource(resultsTransport.location)
         val jira = Jira.Builder(
             nodes = nodes,
             jiraHome = RemoteLocation(
@@ -336,6 +340,7 @@ class DataCenterFormula private constructor(
             address = loadBalancer.uri
         )
             .jmxClients(jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) })
+            .extraMeasurementSources(listOfNotNull(loadBalancerResultsSource))
             .build()
 
         logger.info("$jira is set up, will expire ${jiraStack.expiry}")
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt
@@ -1,40 +1,49 @@
 package com.atlassian.performance.tools.awsinfrastructure.api.jira
 
 import com.atlassian.performance.tools.awsinfrastructure.api.CustomDatasetSource
+import com.atlassian.performance.tools.awsinfrastructure.FailSafeRunnable
 import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation
 import com.atlassian.performance.tools.concurrency.api.submitWithLogContext
 import com.atlassian.performance.tools.infrastructure.api.MeasurementSource
 import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.JmxClient
-import com.atlassian.performance.tools.jvmtasks.api.TaskTimer.time
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
 import java.net.URI
 import java.util.concurrent.Executors
 
+/**
+ * @param extraMeasurementSources source of results/diagnostics of: reverse proxy, Crowd, LDAP, DVCS, DB, Jira plugins
+ * or anything that can be integrated with Jira as part of web application provisioning
+ */
 class Jira private constructor(
     private val nodes: List<StartedNode>,
     val jiraHome: RemoteLocation,
     val database: RemoteLocation,
     val address: URI,
-    val jmxClients: List<JmxClient> = emptyList()
+    val jmxClients: List<JmxClient>,
+    private val extraMeasurementSources: List<MeasurementSource>
 ) : MeasurementSource {
     private val logger: Logger = LogManager.getLogger(this::class.java)
 
     override fun gatherResults() {
-        if (nodes.isEmpty()) {
-            logger.warn("No Jira nodes known to JPT, not downloading node results")
+        val firstNode = nodes.firstOrNull()
+        val measurementSources = extraMeasurementSources + nodes + listOfNotNull(firstNode?.let { AnalyticsLogsSource(it) })
+        if (measurementSources.isEmpty()) {
+            logger.warn("No result sources known to Jira, can't download anything")
             return
         }
         val executor = Executors.newFixedThreadPool(
-            nodes.size.coerceAtMost(4),
+            measurementSources.size.coerceAtMost(4),
             ThreadFactoryBuilder()
                 .setNameFormat("results-gathering-thread-%d")
                 .build()
         )
-        nodes.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } }
-            .forEach { it.get() }
-        time("gather analytics") { nodes.firstOrNull()?.gatherAnalyticLogs() }
+        FailSafeRunnable(
+            measurementSources.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } }
+                .map { Runnable { it.get() } }
+        ).run()
+
         executor.shutdownNow()
     }
 
@@ -55,14 +64,26 @@ class Jira private constructor(
         private val address: URI
     ) {
         private var jmxClients: List<JmxClient> = emptyList()
+        private var extraMeasurementSources: List<MeasurementSource> = emptyList()
+
         fun jmxClients(jmxClients: List<JmxClient>) = apply { this.jmxClients = jmxClients }
+        fun extraMeasurementSources(extraMeasurementSources: List<MeasurementSource>) = apply { this.extraMeasurementSources = extraMeasurementSources }
 
         fun build() = Jira(
             nodes = nodes,
             jiraHome = jiraHome,
             database = database,
             address = address,
-            jmxClients = jmxClients
+            jmxClients = jmxClients,
+            extraMeasurementSources = extraMeasurementSources
         )
     }
+
+    private class AnalyticsLogsSource(
+        private val node: StartedNode
+    ) : MeasurementSource {
+        override fun gatherResults() {
+            node.gatherAnalyticLogs()
+        }
+    }
 }
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt
@@ -2,6 +2,7 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira
 
 import com.atlassian.performance.tools.aws.api.Storage
 import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli
+import com.atlassian.performance.tools.infrastructure.api.MeasurementSource
 import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog
 import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess
 import com.atlassian.performance.tools.ssh.api.Ssh
@@ -15,10 +16,10 @@ class StartedNode(
     private val jiraPath: String,
     private val monitoringProcesses: List<RemoteMonitoringProcess>,
     private val ssh: Ssh
-) {
+) : MeasurementSource {
     private val resultsDirectory = "results"
 
-    fun gatherResults() {
+    override fun gatherResults() {
         ssh.newConnection().use { shell ->
             monitoringProcesses.forEach { it.stop(shell) }
             val nodeResultsDirectory = "$resultsDirectory/'$name'"
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt
@@ -1,7 +1,8 @@
 package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer
 
+import com.atlassian.performance.tools.aws.api.StorageLocation
+import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli
 import com.atlassian.performance.tools.infrastructure.api.Sed
-import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer
 import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff
 import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction
 import com.atlassian.performance.tools.ssh.api.Ssh
@@ -14,7 +15,7 @@ class ApacheProxyLoadBalancer private constructor(
     private val ssh: Ssh,
     ipAddress: String,
     httpPort: Int
-) : LoadBalancer {
+) : DiagnosableLoadBalancer {
 
     override val uri: URI = URI("http://${ipAddress}:$httpPort/")
 
@@ -81,6 +82,15 @@ class ApacheProxyLoadBalancer private constructor(
         connection.execute("echo \"$line\" | sudo tee -a $APACHE_CONFIG_PATH")
     }
 
+    override fun gatherEvidence(location: StorageLocation) {
+        ssh.newConnection().use { connection ->
+            val resultsDir = "/tmp/s3-results"
+            connection.execute("mkdir -p $resultsDir")
+            connection.execute("cp -R /var/log/apache2 $resultsDir")
+            AwsCli().upload(location, connection, resultsDir, Duration.ofMinutes(1))
+        }
+    }
+
     class Builder(
         private val ssh: Ssh
     ) {
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt
@@ -0,0 +1,8 @@
+package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer
+
+import com.atlassian.performance.tools.aws.api.StorageLocation
+import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer
+
+interface DiagnosableLoadBalancer : LoadBalancer {
+    fun gatherEvidence(location: StorageLocation)
+}
diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt
@@ -0,0 +1,23 @@
+package com.atlassian.performance.tools.awsinfrastructure.loadbalancer
+
+import com.atlassian.performance.tools.aws.api.StorageLocation
+import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer
+import com.atlassian.performance.tools.infrastructure.api.MeasurementSource
+
+internal class LoadBalancerMeasurementSource(
+    private val loadBalancer: DiagnosableLoadBalancer,
+    private val target: StorageLocation
+) : MeasurementSource {
+    internal object Extension {
+        fun DiagnosableLoadBalancer.asMeasurementSource(
+            target: StorageLocation
+        ) = LoadBalancerMeasurementSource(
+            loadBalancer = this,
+            target = target
+        )
+    }
+
+    override fun gatherResults() {
+        loadBalancer.gatherEvidence(target)
+    }
+}
diff --git a/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt
@@ -0,0 +1,67 @@
+package com.atlassian.performance.tools.awsinfrastructure
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Test
+
+class FailSafeRunnableTest {
+    @Test
+    fun shouldExecuteAllEvenIfFirstFails() {
+        var executed1 = false
+        var executed2 = false
+        var executed3 = false
+        val runnable = FailSafeRunnable(
+            listOf(
+                Runnable { executed1 = true; throw Exception("Fail 1") },
+                Runnable { executed2 = true; throw Exception("Fail 2") },
+                Runnable { executed3 = true; throw Exception("Fail 3") }
+            )
+        )
+
+        try {
+            runnable.run()
+        } catch (e: Exception) {
+            // Expected and ignored, so that we can go to asserts
+        }
+
+        assertThat(executed1).isTrue()
+        assertThat(executed2).isTrue()
+        assertThat(executed3).isTrue()
+    }
+
+    @Test
+    fun shouldThrowAllFailures() {
+        val runnable = FailSafeRunnable(
+            listOf(
+                Runnable { throw Exception("Banana") },
+                Runnable { throw Exception("Apple") },
+                Runnable { throw Exception("Pear") },
+                Runnable { throw Exception("Peach") }
+            )
+        )
+
+        val exception = try {
+            runnable.run()
+            null
+        } catch (e: Exception) {
+            e
+        }
+
+        val allExceptions = listOf(exception!!) + exception.suppressed.toList()
+        val allMessages = allExceptions.map { it.message }
+        assertThat(allMessages).contains("Banana", "Apple", "Pear", "Peach")
+    }
+
+
+    @Test
+    fun shouldExecuteAll() {
+        val allIndexes = Array(20) { it }
+        val finishedIndexes = mutableListOf<Int>()
+        val runnable = FailSafeRunnable(
+            allIndexes.map { index -> Runnable { finishedIndexes.add(index) } }
+        )
+
+        runnable.run()
+
+        assertThat(finishedIndexes).contains(*allIndexes)
+    }
+}
