Improve sidekiq-unique-jobs orphan lock cleanup

Configure reaper with resurrector enabled, add unique jobs middleware,
and add lock to SyncIssuesWorker. Add rake tasks for listing and
clearing locks manually, with weekly cleanup cron.

Author: andrew

app.json

@@ -23,6 +23,10 @@
     {
       "command": "bundle exec rake hosts:sync_all",
       "schedule": "0 0 * * *"
+    },
+    {
+      "command": "bundle exec rake sidekiq:unique_jobs:clear",
+      "schedule": "0 0 * * 0"
     }
   ]
 } 

app/sidekiq/sync_issues_worker.rb

@@ -2,7 +2,7 @@ class SyncIssuesWorker
   include Sidekiq::Worker
   include Sidekiq::Status::Worker
 
-  sidekiq_options queue: :default
+  sidekiq_options queue: :default, lock: :until_executed, lock_expiration: 1.day.to_i
 
   def perform(job_id)
     Job.find_by_id!(job_id).perform_issue_syncing

config/sidekiq.rb

@@ -1,16 +1,40 @@
 require 'sidekiq'
 require 'sidekiq-status'
 
+SidekiqUniqueJobs.configure do |config|
+  config.reaper          = :ruby
+  config.reaper_count    = 2_000
+  config.reaper_interval = 60
+  config.reaper_timeout  = 15
+
+  config.reaper_resurrector_enabled  = true
+  config.reaper_resurrector_interval = 300
+end
+
 Sidekiq.configure_client do |config|
   config.logger = Rails.logger if Rails.env.test?
-  # accepts :expiration (optional)
+  config.redis = { ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE } }
+
   Sidekiq::Status.configure_client_middleware config, expiration: 60.minutes.to_i
+
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
 end
 
 Sidekiq.configure_server do |config|
-  # accepts :expiration (optional)
-  Sidekiq::Status.configure_server_middleware config, expiration: 60.minutes.to_i
+  config.redis = { ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE } }
 
-  # accepts :expiration (optional)
+  Sidekiq::Status.configure_server_middleware config, expiration: 60.minutes.to_i
   Sidekiq::Status.configure_client_middleware config, expiration: 60.minutes.to_i
-end
+
+  config.client_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Client
+  end
+
+  config.server_middleware do |chain|
+    chain.add SidekiqUniqueJobs::Middleware::Server
+  end
+
+  SidekiqUniqueJobs::Server.configure(config)
+end

lib/tasks/sidekiq.rake

@@ -0,0 +1,53 @@
+namespace :sidekiq do
+  namespace :unique_jobs do
+    desc "List orphaned unique job locks"
+    task list: :environment do
+      digests = SidekiqUniqueJobs::Digests.new
+      entries = digests.entries(pattern: "*", count: 10_000)
+      if entries.empty?
+        puts "No unique job locks found."
+      else
+        puts "#{entries.size} unique job lock(s):"
+        entries.each do |digest, score|
+          time = Time.at(score).utc rescue score
+          puts "  #{digest} (scored at #{time})"
+        end
+      end
+    end
+
+    desc "Clear all unique job locks"
+    task clear: :environment do
+      digests = SidekiqUniqueJobs::Digests.new
+      entries = digests.entries(pattern: "*", count: 10_000)
+      if entries.empty?
+        puts "No unique job locks to clear."
+      else
+        puts "Clearing #{entries.size} unique job lock(s)..."
+        digests.delete_by_pattern("*", count: 10_000)
+        puts "Done."
+      end
+
+      expiring = SidekiqUniqueJobs::ExpiringDigests.new
+      expiring_entries = expiring.entries(pattern: "*", count: 10_000)
+      unless expiring_entries.empty?
+        puts "Clearing #{expiring_entries.size} expiring digest(s)..."
+        expiring.delete_by_pattern("*", count: 10_000)
+        puts "Done."
+      end
+    end
+
+    desc "Clear unique job locks matching a pattern (e.g. PATTERN='*SyncIssues*')"
+    task clear_matching: :environment do
+      pattern = ENV.fetch("PATTERN") { abort "Usage: rake sidekiq:unique_jobs:clear_matching PATTERN='*SyncIssues*'" }
+      digests = SidekiqUniqueJobs::Digests.new
+      entries = digests.entries(pattern: pattern, count: 10_000)
+      if entries.empty?
+        puts "No locks matching '#{pattern}'."
+      else
+        puts "Clearing #{entries.size} lock(s) matching '#{pattern}'..."
+        digests.delete_by_pattern(pattern, count: 10_000)
+        puts "Done."
+      end
+    end
+  end
+end

test/tasks/sidekiq_unique_jobs_rake_test.rb

@@ -0,0 +1,36 @@
+require "test_helper"
+require "rake"
+
+class SidekiqUniqueJobsRakeTest < ActiveSupport::TestCase
+  setup do
+    Rails.application.load_tasks unless Rake::Task.task_defined?("sidekiq:unique_jobs:list")
+    @digests = SidekiqUniqueJobs::Digests.new
+  end
+
+  test "list task runs without error" do
+    assert_nothing_raised do
+      capture_io { Rake::Task["sidekiq:unique_jobs:list"].execute }
+    end
+  end
+
+  test "clear task runs without error" do
+    assert_nothing_raised do
+      capture_io { Rake::Task["sidekiq:unique_jobs:clear"].execute }
+    end
+  end
+
+  test "clear_matching task aborts without PATTERN" do
+    assert_raises(SystemExit) do
+      capture_io { Rake::Task["sidekiq:unique_jobs:clear_matching"].execute }
+    end
+  end
+
+  test "clear_matching task runs with PATTERN" do
+    ENV["PATTERN"] = "*"
+    assert_nothing_raised do
+      capture_io { Rake::Task["sidekiq:unique_jobs:clear_matching"].execute }
+    end
+  ensure
+    ENV.delete("PATTERN")
+  end
+end