Add Capistrano task to remove application from the load balancer. (#1048)

* Add Capistrano task to remove application from the load balancer.

Closes #1041

* Still log.

Author: tpendragon

Gemfile

@@ -61,7 +61,7 @@ gem "datacite", github: "sul-dlss/datacite-ruby", branch: "main"
 gem "dogstatsd-ruby"
 gem "flipflop"
 gem "google-protobuf", "~> 3.25"
-gem "health-monitor-rails"
+gem "health-monitor-rails", "12.4.0"
 gem "honeybadger"
 gem "mailcatcher"
 gem "net-http-persistent"

Gemfile.lock

@@ -217,7 +217,7 @@ GEM
     google-protobuf (3.25.5-x86_64-linux)
     hashdiff (1.1.0)
     hashie (5.0.0)
-    health-monitor-rails (12.1.0)
+    health-monitor-rails (12.4.0)
       railties (>= 6.1)
     honeybadger (5.10.2)
     i18n (1.14.6)
@@ -557,7 +557,7 @@ DEPENDENCIES
   ffaker
   flipflop
   google-protobuf (~> 3.25)
-  health-monitor-rails
+  health-monitor-rails (= 12.4.0)
   honeybadger
   importmap-rails
   jbuilder

config/deploy.rb

@@ -21,6 +21,37 @@
 end
 after "deploy:log_revision", "write_version"
 
+namespace :application do
+  # You can/ should apply this command to a single host
+  # cap --hosts=tigerdata-staging1.lib.princeton.edu staging application:remove_from_nginx
+  desc "Marks the server(s) to be removed from the loadbalancer"
+  task :remove_from_nginx do
+    count = 0
+    on roles(:app) do
+      count += 1
+    end
+    if count > (roles(:app).length / 2)
+      raise "You must run this command on no more than half the servers utilizing the --hosts= switch"
+    end
+    on roles(:app) do
+      within release_path do
+        execute :touch, "public/remove-from-nginx"
+      end
+    end
+  end
+
+  # You can/ should apply this command to a single host
+  # cap --hosts=tigerdata-staging1.lib.princeton.edu staging application:serve_from_nginx
+  desc "Marks the server(s) to be added back to the loadbalancer"
+  task :serve_from_nginx do
+    on roles(:app) do
+      within release_path do
+        execute :rm, "-f public/remove-from-nginx"
+      end
+    end
+  end
+end
+
 # rubocop:disable Rails/Output
 namespace :mailcatcher do
   desc "Opens Mailcatcher Consoles"

config/initializers/health_monitor.rb

@@ -3,6 +3,9 @@
   HealthMonitor.configure do |config|
     config.cache
 
+    config.file_absence.configure do |file_config|
+      file_config.filename = "public/remove-from-nginx"
+    end
     # Mediaflux check
     config.add_custom_provider(MediafluxStatus)
 
@@ -11,7 +14,9 @@
 
     config.error_callback = proc do |e|
       Rails.logger.error "Health check failed with: #{e.message}"
-      Honeybadger.notify(e)
+      unless e.is_a?(HealthMonitor::Providers::FileAbsenceException)
+        Honeybadger.notify(e)
+      end
     end
   end
 end