Merge pull request #1902 from OpenC3/kill_on_thread_crash

Kill Microservice on any Thread Crash

Author: ryanmelt

openc3-cosmos-script-runner-api/Gemfile

@@ -33,6 +33,7 @@ group :development, :test do
   # Call 'byebug' anywhere in the code to stop execution and get a debugger console
   gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
   gem 'rspec-rails', '~> 7.0'
+  gem 'rspec'
   gem 'simplecov', '~> 0.20'
   gem 'simplecov-cobertura', '~> 2.1'
 end

openc3-cosmos-script-runner-api/spec/spec_helper.rb

@@ -34,6 +34,8 @@
 # the additional setup, and require it from the spec files that actually need
 # it.
 
+require 'rspec'
+
 # NOTE: You MUST require simplecov before anything else!
 if !ENV['OPENC3_NO_SIMPLECOV']
   require 'simplecov'

openc3/lib/openc3/microservices/cleanup_microservice.rb

@@ -100,4 +100,8 @@ def shutdown
   end
 end
 
-OpenC3::CleanupMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::CleanupMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/decom_microservice.rb

@@ -48,6 +48,7 @@ def start
         @logger.error "#{@microservice_name}: Limits Response thread died: #{e.formatted}"
         raise e
       end
+      ThreadManager.instance.register(@thread, stop_object: self)
     end
 
     def stop
@@ -251,4 +252,8 @@ def limits_change_callback(packet, item, old_limits_state, value, log_change)
   end
 end
 
-OpenC3::DecomMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::DecomMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/interface_microservice.rb

@@ -70,6 +70,7 @@ def start
         @logger.error "#{@interface.name}: Command handler thread died: #{e.formatted}"
         raise e
       end
+      ThreadManager.instance.register(@thread, stop_object: self)
     end
 
     def stop()
@@ -346,6 +347,7 @@ def start
         @logger.error "#{@router.name}: Telemetry handler thread died: #{e.formatted}"
         raise e
       end
+      ThreadManager.instance.register(@thread, stop_object: self)
     end
 
     def stop
@@ -813,4 +815,8 @@ def graceful_kill
   end
 end
 
-OpenC3::InterfaceMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::InterfaceMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/log_microservice.rb

@@ -142,4 +142,8 @@ def shutdown
   end
 end
 
-OpenC3::LogMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::LogMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/microservice.rb

@@ -30,6 +30,7 @@
 OpenC3.require_file 'openc3/utilities/secrets'
 OpenC3.require_file 'openc3/utilities/sleeper'
 OpenC3.require_file 'openc3/utilities/open_telemetry'
+OpenC3.require_file 'openc3/utilities/thread_manager'
 OpenC3.require_file 'openc3/models/microservice_model'
 OpenC3.require_file 'openc3/models/microservice_status_model'
 OpenC3.require_file 'tmpdir'
@@ -49,22 +50,27 @@ class Microservice
     def self.run(name = nil)
       name = ENV['OPENC3_MICROSERVICE_NAME'] unless name
       microservice = self.new(name)
-      begin
-        MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
-        microservice.state = 'RUNNING'
-        microservice.run
-        microservice.state = 'FINISHED'
-      rescue Exception => e
-        if SystemExit === e or SignalException === e
-          microservice.state = 'KILLED'
-        else
-          microservice.error = e
-          microservice.state = 'DIED_ERROR'
-          Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
+      thread = Thread.new do
+        begin
+          MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
+          microservice.state = 'RUNNING'
+          microservice.run
+          microservice.state = 'FINISHED'
+        rescue Exception => e
+          if SystemExit === e or SignalException === e
+            microservice.state = 'KILLED'
+          else
+            microservice.error = e
+            microservice.state = 'DIED_ERROR'
+            Logger.fatal("Microservice #{name} dying from exception\n#{e.formatted}")
+          end
+        ensure
+          MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
         end
-      ensure
-        MicroserviceStatusModel.set(microservice.as_json(:allow_nan => true), scope: microservice.scope)
       end
+      ThreadManager.instance.register(thread, shutdown_object: microservice)
+      ThreadManager.instance.monitor
+      ThreadManager.instance.shutdown
     end
 
     def as_json(*a)
@@ -192,6 +198,7 @@ def initialize(name, is_plugin: false)
           @logger.error "#{@name} status thread died: #{e.formatted}"
           raise e
         end
+        ThreadManager.instance.register(@microservice_status_thread)
       end
     end
 

openc3/lib/openc3/microservices/multi_microservice.rb

@@ -18,14 +18,15 @@
 
 require 'openc3/microservices/microservice'
 require 'openc3/topics/topic'
+require 'openc3/utilities/thread_manager'
 
 module OpenC3
   class MultiMicroservice < Microservice
     def run
       @threads = []
       ARGV.each do |microservice_name|
         microservice_model = MicroserviceModel.get_model(name: microservice_name, scope: @scope)
-        @threads << Thread.new do
+        thread = Thread.new do
           cmd_line = microservice_model.cmd.join(' ')
           split_cmd_line = cmd_line.split(' ')
           filename = nil
@@ -42,21 +43,15 @@ def run
           klass = filename.filename_to_class_name.to_class
           klass.run(microservice_model.name)
         end
+        ThreadManager.instance.register(thread)
       end
-      @threads.each do |thread|
-        thread.join
-      end
-    end
-
-    def shutdown
-      super()
-      if @threads
-        @threads.each do |thread|
-          thread.join
-        end
-      end
+      ThreadManager.instance.monitor
+      ThreadManager.instance.shutdown
     end
   end
 end
-
-OpenC3::MultiMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::MultiMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/periodic_microservice.rb

@@ -87,4 +87,8 @@ def shutdown
   end
 end
 
-OpenC3::PeriodicMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::PeriodicMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end

openc3/lib/openc3/microservices/reducer_microservice.rb

@@ -633,4 +633,8 @@ def extract_entry_samples(packet)
   end
 end
 
-OpenC3::ReducerMicroservice.run if __FILE__ == $0
+if __FILE__ == $0
+  OpenC3::ReducerMicroservice.run
+  ThreadManager.instance.shutdown
+  ThreadManager.instance.join
+end