Implement graceful shutdown for recording importer on SIGTERM/SIGINT (#1259)

* Implement graceful shutdown for recording importer on SIGTERM/SIGINT

* scalelite-recording-importer: implement stop timeout and exec stop command in systemd service file

Author: michelleDeko

lib/tasks/recordings.rake

@@ -10,18 +10,26 @@ namespace :recordings do
     # Setup AWS S3 Client if enabled (auto looks up the env variables it needs)
     client = Aws::S3::Client.new if ENV['S3_RECORDING']
 
+    # On SIGTERM/SIGINT only set a flag; the in-progress import is allowed to
+    # finish before the loop exits, so a running extraction is never interrupted.
+    shutdown = false
+    %w[TERM INT].each do |sig|
+      Signal.trap(sig) { shutdown = true }
+    end
+
     loop do
       Dir.glob("#{dir}/*.tar").each do |file|
+        break if shutdown
         Rails.logger.debug { "Found #{file}" }
         RecordingImporter.import(file, client)
       rescue StandardError => e
         Rails.logger.error("Failed to import recording: #{e}")
         sleep(args.latency.to_f)
       end
+      break if shutdown
       sleep(args.latency.to_f)
     end
-  rescue SignalException => e
-    Rails.logger.info("Exiting recording importer on signal: #{e}")
+    Rails.logger.info('Exiting recording importer gracefully')
   end
 
   desc 'Search through the recordings and move unpublished recordings to the correct folder'

systemd/scalelite-recording-importer.service

@@ -8,10 +8,12 @@ After=scalelite-api.service
 After=remote-fs.target
 [Service]
 EnvironmentFile=/etc/default/scalelite
+TimeoutStopSec=140
 ExecStartPre=-/usr/bin/docker kill scalelite-recording-importer
 ExecStartPre=-/usr/bin/docker rm scalelite-recording-importer
 ExecStartPre=/usr/bin/docker pull blindsidenetwks/scalelite:${SCALELITE_TAG}-recording-importer
-ExecStart=/usr/bin/docker run --log-opt max-size=10m --log-opt max-file=5 --name scalelite-recording-importer --env-file /etc/default/scalelite --network scalelite --mount type=bind,source=${SCALELITE_RECORDING_DIR},target=/var/bigbluebutton blindsidenetwks/scalelite:${SCALELITE_TAG}-recording-importer
+ExecStart=/usr/bin/docker run --log-opt max-size=10m --log-opt max-file=5 --stop-timeout=120 --name scalelite-recording-importer --env-file /etc/default/scalelite --network scalelite --mount type=bind,source=${SCALELITE_RECORDING_DIR},target=/var/bigbluebutton blindsidenetwks/scalelite:${SCALELITE_TAG}-recording-importer
+ExecStop=/usr/bin/docker stop scalelite-recording-importer
 ExecStartPost=/usr/bin/docker image prune -f
 [Install]
 WantedBy=scalelite.target