feat: Added concurrency to Hyperproof sync.

Author: jamesiarmes

README.md

@@ -1,5 +1,5 @@
 # Code for America Security Controls
 
-This repository contains the configuration for our AWS security controls. These
-configurations ensure that our AWS accounts are secure and meet our compliance
+This repository contains the configuration and automation for our security
+controls. These ensure that our organization is secure and meets our compliance
 requirements.

hyperproof/Gemfile.lock

@@ -6,6 +6,7 @@ PATH
       aws-sdk-configservice (~> 1.128)
       aws-sdk-rds (~> 1.276)
       aws-sdk-resourceexplorer2 (~> 1.36)
+      concurrent-ruby (~> 1.3)
       configsl (~> 1.0)
       csv (~> 3.3)
       faraday (~> 2.13)

hyperproof/bin/hyperproof

@@ -11,17 +11,37 @@ module CfaSecurityControls
   module Hyperproof
     # CLI tool for the CFA Security Controls Hyperproof integration.
     class CLI < Thor
+      def self.exit_on_failure?
+        true
+      end
+
       desc 'collect', 'Collect and sync all proofs to Hyperproof'
       def collect
         say 'Collecting proofs...', :green
-        CfaSecurityControls::Hyperproof.run
-        say 'All proofs collected and synced to Hyperproof.', :green
+        results = CfaSecurityControls::Hyperproof.run
+        if results.values.any?(false)
+          say_error "#{results.values.tally[false]} proofs failed to collect.", :red
+          exit 1
+        end
+
+        say "Collected and synced #{bold(results.length)} proofs to Hypersync.", :green
       end
 
       desc 'version', 'Show the current version'
       def version
         say CfaSecurityControls::Hyperproof::VERSION
       end
+
+      # Helper methods that will not be exposed as commands.
+      no_commands do
+        # Bold a string for terminal output.
+        #
+        # @param string [String] The string to bold.
+        # @return [String] The bolded string.
+        def bold(string)
+          "\e[1m#{string}\e[22m"
+        end
+      end
     end
   end
 end

hyperproof/cfa-security-controls-hyperproof.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'aws-sdk-configservice', '~> 1.128'
   s.add_dependency 'aws-sdk-rds', '~> 1.276'
   s.add_dependency 'aws-sdk-resourceexplorer2', '~> 1.36'
+  s.add_dependency 'concurrent-ruby', '~> 1.3'
   s.add_dependency 'configsl', '~> 1.0'
   s.add_dependency 'csv', '~> 3.3'
   s.add_dependency 'faraday', '~> 2.13'

hyperproof/lib/cfa_security_controls/hyperproof.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'concurrent-ruby'
+
 require_relative 'hyperproof/config'
 require_relative 'hyperproof/entities/label'
 require_relative 'hyperproof/entities/proof'
@@ -30,16 +32,43 @@ def self.config(config = nil)
     def self.run
       Dir.mktmpdir do |dir|
         config.logger.info("Writing proofs to #{dir}")
+
         writer = Writer.new(dir)
-        Proofs.proofs.map do |klass|
-          proof = klass.new
-          filename = collect_proof(proof, writer)
-          Entities::Proof.new(File.basename(filename), label: proof_label(proof))
-                         .create(filename)
+        futures = Proofs.proofs.map do |klass|
+          Concurrent::Promises.future(executor: config.thread_pool) do
+            create_proof(klass.new, writer)
+          end.run
         end
+
+        # Wait for all futures to complete and collect results.
+        Concurrent::Promises.zip(*futures).value!.to_h
       end
     end
 
+    # Create the proof in Hyperproof.
+    #
+    # @param proof [Proofs::Proof] The proof to create.
+    # @param writer [Writer] The writer to use for formatting the proof.
+    # @return [Array] A promise response containing the proof name and its ID.
+    private_class_method def self.create_proof(proof, writer)
+      filename = collect_proof(proof, writer)
+      entity = Entities::Proof.new(File.basename(filename), label: proof_label(proof))
+      entity.create(filename)
+      [proof.name, entity.id]
+    rescue StandardError => e
+      handle_exception(proof, e)
+    end
+
+    # Handle exceptions that occur during proof collection.
+    #
+    # @param proof [Proofs::Proof] The proof that was being processed.
+    # @param error [StandardError] The error that occurred.
+    # @return [Array] Valid promise repose representing a failure.
+    private_class_method def self.handle_exception(proof, error)
+      config.logger.error("An error occurred: #{error.message}")
+      [proof.name, false]
+    end
+
     # Collect evidence for a specific proof.
     #
     # @param proof [Proofs::Proof] The proof to collect evidence for.

hyperproof/lib/cfa_security_controls/hyperproof/config.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'concurrent-ruby'
 require 'configsl'
 
 module CfaSecurityControls
@@ -13,6 +14,9 @@ class Config < ConfigSL::Config
       option :aptible_password, type: String
       option :hyperproof_client_id, type: String, required: true
       option :hyperproof_client_secret, type: String, required: true
+      option :threads_min, type: Integer, default: 1
+      option :threads_max, type: Integer, default: Concurrent.processor_count
+      option :theads_queue_size, type: Integer, default: 10
 
       def initialize(params = {})
         super
@@ -22,6 +26,15 @@ def initialize(params = {})
       def logger
         @logger ||= Logger.new($stdout, level: log_level)
       end
+
+      def thread_pool
+        @thread_pool ||= Concurrent::ThreadPoolExecutor.new(
+          min_threads: threads_min,
+          max_threads: threads_max,
+          max_queue: theads_queue_size,
+          fallback_policy: :caller_runs
+        )
+      end
     end
   end
 end