Merge pull request #18 from TalkingQuickly/feature/refactor

Feature/refactor

Author: TalkingQuickly

CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+
+## 5.0.1 (March 2021)
+
+- Adds full support for deploy (but not config creation) without sudo access
+- Refactor config files to use single array of hashes with flags
+- Refactor Sidekiq and Monit configurations to copy files directly rather than using symlinks to avoid potential root access leak
+- Fixes bug where object identifier was outputted in logs rather than filename
+- Fixes nginx not being reloaded after `setup_config` due to shared log directory not yet existing
+
+## 5.0.0 (March 2021)
+
+- Full overhaul to support Rails 6 and Ubuntu 20.04

capistrano-cookbook.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'capistrano', '~>  3.16'
   spec.add_dependency 'capistrano3-puma', '~>  5.0.4'
+  spec.add_dependency 'capistrano-sidekiq', '~> 2.0'
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"

lib/capistrano/cookbook.rb

@@ -9,6 +9,7 @@ module Cookbook
     require 'capistrano/cookbook/run_tests'
     require 'capistrano/cookbook/setup_config'
     require 'capistrano/cookbook/create_database'
-    require 'capistrano/cookbook/systemd'
+    require 'capistrano/cookbook/puma_systemd'
+    require 'capistrano/cookbook/sidekiq_systemd'
   end
 end

lib/capistrano/cookbook/helpers/setup_config_values.rb

@@ -1,59 +1,46 @@
 module Capistrano
   module Cookbook
     class SetupConfigValues
-      def symlinks
-        fetch(:symlinks) || symlinks_defaults
-      end
-
-      def executable_config_files
-        fetch(:executable_config_files) || executable_config_files_defaults
-      end
-
       def config_files
         fetch(:config_files) || config_files_defaults
       end
 
       private
 
-      def symlinks_defaults
+      def config_files_defaults
         base = [
           {
-            source: "log_rotation",
-            link: "/etc/logrotate.d/{{full_app_name}}"
+            source: 'log_rotation',
+            destination: "/etc/logrotate.d/#{fetch(:full_app_name)}",
+            executable: false,
+            as_root: true 
+          },
+          {
+            source: 'database.example.yml',
+            destination: "#{shared_path}/config/database.example.yml",
+            executable: false,
+            as_root: false
           }
         ]
+
         return base unless sidekiq_enabled?
 
         base + [
           {
-            source: "sidekiq.service.capistrano",
-            link: "/etc/systemd/system/#{fetch(:sidekiq_service_unit_name)}.service"
+            source: 'sidekiq.service.capistrano',
+            destination: "/home/#{fetch(:deploy_user)}/.config/systemd/user/#{fetch(:sidekiq_service_unit_name)}.service",
+            executable: false,
+            as_root: false
           },
           {
             source: "sidekiq_monit",
-            link: "/etc/monit/conf.d/#{fetch(:full_app_name)}_sidekiq.conf"
+            destination: "/etc/monit/conf.d/#{fetch(:full_app_name)}_sidekiq.conf",
+            executable: false,
+            as_root: true
           }
         ]
       end
 
-      def executable_config_files_defaults
-        %w(
-        )
-      end
-
-      def config_files_defaults
-        base = %w(
-          database.example.yml
-          log_rotation
-        )
-        return base unless sidekiq_enabled?
-
-        base + %w(
-          sidekiq.service.capistrano
-          sidekiq_monit
-        )
-      end
-
       def sidekiq_enabled?
         defined?(Capistrano::Sidekiq) == 'constant' && Capistrano::Sidekiq.class == Class
       end

lib/capistrano/cookbook/helpers/template.rb lib/capistrano/cookbook/helpers/smart_template.rb

@@ -1,3 +1,6 @@
+require 'securerandom'
+require 'stringio'
+
 # will first try and copy the file:
 # config/deploy/#{full_app_name}/#{from}.erb
 # to:
@@ -11,13 +14,12 @@
 # ones to be over-ridden
 # if the target file name is the same as the source then
 # the second parameter can be left out
-def smart_template(from, to=nil)
-  to ||= from
-  full_to_path = "#{shared_path}/config/#{to}"
+def smart_template(from, to, as_root=false)
   if from_erb_path = template_file(from)
     from_erb = StringIO.new(ERB.new(File.read(from_erb_path)).result(binding))
-    upload! from_erb, full_to_path
-    info "copying: #{from_erb} to: #{full_to_path}"
+    upload!(from_erb, to) unless as_root
+    sudo_upload!(from_erb, to) if as_root
+    info "copying: #{from} to: #{to}"
   else
     error "error #{from} not found"
   end
@@ -35,3 +37,14 @@ def template_file(name)
   end
   return nil
 end
+
+def sudo_upload!(file_path, remote_path, mode: '644', owner: 'root:root')
+  tmp_path = "/tmp/#{SecureRandom.uuid}"
+
+  upload!(file_path, tmp_path)
+
+  execute(:sudo, :mkdir, '-p', File.dirname(remote_path))
+  execute(:sudo, :mv, '-f', tmp_path, remote_path)
+  execute(:sudo, :chmod, mode, remote_path)
+  execute(:sudo, :chown, owner, remote_path)
+end

lib/capistrano/cookbook/puma_systemd.rb

@@ -0,0 +1 @@
+load File.expand_path("tasks/puma_systemd.cap", File.dirname(__FILE__))

lib/capistrano/cookbook/sidekiq_systemd.rb

@@ -0,0 +1 @@
+load File.expand_path("tasks/sidekiq_systemd.cap", File.dirname(__FILE__))

lib/capistrano/cookbook/systemd.rb

@@ -3,7 +3,7 @@ namespace :nginx do
     desc "#{task } Nginx"
     task task_name do
       on roles(:app), in: :sequence, wait: 5 do
-        sudo "/etc/init.d/nginx #{task_name}"
+        sudo "systemctl #{task_name} nginx"
       end
     end
   end

lib/capistrano/cookbook/tasks/nginx.cap

@@ -6,8 +6,8 @@ namespace :puma do
         if fetch(:puma_systemctl_user) == :system
           sudo "#{fetch(:puma_systemctl_bin)} reload-or-restart #{fetch(:puma_service_unit_name)}"
         else
-          execute "#{fetch(:puma_systemctl_bin)}", "--user", "reload", fetch(:puma_service_unit_name)
           execute :loginctl, "enable-linger", fetch(:puma_lingering_user) if fetch(:puma_enable_lingering)
+          execute "#{fetch(:puma_systemctl_bin)}", "--user", "reload-or-restart", fetch(:puma_service_unit_name)
         end
       end
     end

lib/capistrano/cookbook/tasks/systemd.cap lib/capistrano/cookbook/tasks/puma_systemd.cap

@@ -1,7 +1,7 @@
 require 'capistrano/dsl'
 require 'capistrano/cookbook/helpers/setup_config_values'
 require 'capistrano/cookbook/helpers/substitute_strings'
-require 'capistrano/cookbook/helpers/template'
+require 'capistrano/cookbook/helpers/smart_template'
 require 'capistrano/cookbook/nginx'
 require 'capistrano/cookbook/monit'
 require 'securerandom'
@@ -13,22 +13,24 @@ namespace :deploy do
     on roles(:app) do
       # make the config dir
       execute :mkdir, "-p #{shared_path}/config"
+      execute :mkdir, "-p /home/#{fetch(:deploy_user)}/.config/systemd/user"
 
       # config files to be uploaded to shared/config, see the
       # definition of smart_template for details of operation.
       conf.config_files.each do |file|
-        smart_template file
+        smart_template(file[:source], file[:destination], file[:as_root])
+        execute(:chmod, "+x #{file[:destination]}") if file[:executable]
       end
 
       # which of the above files should be marked as executable
-      conf.executable_config_files.each do |file|
-        execute :chmod, "+x #{shared_path}/config/#{file}"
-      end
+      # conf.executable_config_files.each do |file|
+      #   execute :chmod, "+x #{shared_path}/config/#{file}"
+      # end
 
       # symlink stuff which should be... symlinked
-      conf.symlinks.each do |symlink|
-        sudo "ln -nfs #{shared_path}/config/#{symlink[:source]} #{sub_strings(symlink[:link])}"
-      end
+      # conf.symlinks.each do |symlink|
+      #   sudo "ln -nfs #{shared_path}/config/#{symlink[:source]} #{sub_strings(symlink[:link])}"
+      # end
 
       if File.exists?(File.join('config', 'master.key'))
         upload! File.join('config', 'master.key'), File.join(shared_path, 'config', 'master.key')
@@ -41,13 +43,23 @@ end
 # remove the default nginx configuration as it will tend to conflict with our configs
 before 'deploy:setup_config', 'nginx:remove_default_vhost'
 
+# make sure that shared directories etc exist before running otherwise the
+# initial nginx reload won't work because of the nginx log file directory path
+# not existing
+before 'deploy:setup_config', 'deploy:check:directories'
+before 'deploy:setup_config', 'deploy:check:linked_dirs'
+
 # After setup config has generated and setup initial files, run the Capistrano Puma
 # tasks responsible for uploading config files. Note that `setup_config` creates overrides
 # for these in `config/deploy/templates` so we're not using the default ones from the gem
 after 'deploy:setup_config', 'puma:config'
 after 'deploy:setup_config', 'puma:nginx_config'
 after 'deploy:setup_config', 'puma:monit:config'
 after 'deploy:setup_config', 'puma:systemd:config'
+after 'deploy:setup_config', 'puma:systemd:enable'
+
+# Enable the sidekiq systemd service so that it's started automatically on (re)boot
+after 'deploy:setup_config', 'sidekiq:systemd:enable' if (defined?(Capistrano::Sidekiq) == 'constant' && Capistrano::Sidekiq.class == Class)
 
 # reload nginx to it will pick up any modified vhosts from setup_config
 after 'deploy:setup_config', 'nginx:reload'

lib/capistrano/cookbook/tasks/setup_config.cap

@@ -0,0 +1,15 @@
+namespace :sidekiq do
+  namespace :systemd do
+    desc 'Install systemd sidekiq service'
+    task :enable do
+      on roles fetch(:sidekiq_roles) do |role|
+        if fetch(:sidekiq_service_unit_user) == :system
+          execute :sudo, :systemctl, "enable", fetch(:sidekiq_service_unit_name)
+        else
+          execute :systemctl, "--user", "enable", fetch(:sidekiq_service_unit_name)
+          execute :loginctl, "enable-linger", fetch(:sidekiq_systemctl_user) if fetch(:sidekiq_enable_lingering)
+        end
+      end
+    end
+  end
+end

lib/capistrano/cookbook/tasks/sidekiq_systemd.cap

@@ -1,5 +1,5 @@
 module Capistrano
   module Cookbook
-    VERSION = "5.0.0"
+    VERSION = "5.0.1"
   end
 end

lib/capistrano/cookbook/version.rb

@@ -13,20 +13,19 @@ set :rbenv_ruby, '3.0.0'
 set :rbenv_prefix, "RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec"
 set :rbenv_map_bins, %w{rake gem bundle ruby rails}
 
-<% if @generate_sidekiq %>
-# Setup sidekiq, make sure we run sidekiq as our deployment user, otherwise
-# it will default to root which a) is insecture and b) will lead to lots of
-# strange permissions issues
-set :sidekiq_service_unit_user, :system
-set :sidekiq_user, fetch(:deploy_user)
-<% end %>
-
 # setup puma to operate in clustered mode, required for zero downtime deploys
 set :puma_preload_app, false
 set :puma_init_active_record, true
 set :puma_workers, 3
+set :puma_systemctl_user, fetch(:deploy_user)
+set :puma_enable_lingering, true
+
+<% if @generate_sidekiq %>
+set :sidekiq_systemctl_user, fetch(:deploy_user)
+set :sidekiq_enable_lingering, true
+<% end %>
 
-# how many old releases do we want to keep, not much
+# how many old releases do we want to keep
 set :keep_releases, 5
 
 # Directories that should be linked to the shared folder

lib/generators/capistrano/reliably_deploying_rails/templates/deploy.rb.erb

@@ -14,7 +14,7 @@ set :full_app_name, "#{fetch(:application)}_#{fetch(:stage)}"
 <% if @generate_sidekiq %>
 # Name sidekiq systemd service after the app and stage name so that
 # multiple apps and stages can co-exist on the same machine if needed
-set :sidekiq_service_unit_name, "#{fetch(:full_app_name)}_sidekiq"
+set :sidekiq_service_unit_name, "sidekiq_#{fetch(:full_app_name)}"
 <% end %>
 
 server '<%= @production_server_address %>', user: 'deploy', roles: %w{web app db}, primary: true

lib/generators/capistrano/reliably_deploying_rails/templates/production.rb.erb

@@ -1,7 +1,4 @@
-# Monit configuration for Puma
-# Service name: <%= puma_monit_service_name %>
-#
 check process <%= puma_monit_service_name %>
   with pidfile "<%= fetch(:puma_pid) %>"
-  start program = "/usr/bin/systemctl start <%= fetch(:puma_service_unit_name) %>"
-  stop program = "/usr/bin/systemctl stop <%= fetch(:puma_service_unit_name) %>"
+  start program = "/bin/bash -c 'XDG_RUNTIME_DIR=/run/user/$(id -u) /usr/bin/systemctl start --user <%= fetch(:puma_service_unit_name) %>'" as uid "<%= fetch(:puma_systemctl_user) %>" and gid "<%= fetch(:puma_systemctl_user) %>"
+  stop program = "/bin/bash -c 'XDG_RUNTIME_DIR=/run/user/$(id -u) /usr/bin/systemctl stop --user <%= fetch(:puma_service_unit_name) %>'" as uid "<%= fetch(:puma_systemctl_user) %>" and gid "<%= fetch(:puma_systemctl_user) %>"

lib/generators/capistrano/reliably_deploying_rails/templates/puma_monit.conf.erb

@@ -1,5 +1,3 @@
-<% # Adapted from: https://github.com/seuros/capistrano-sidekiq/blob/master/lib/generators/capistrano/sidekiq/systemd/templates/sidekiq.service.capistrano.erb %>
-
 [Unit]
 Description=sidekiq for <%= "#{fetch(:application)} (#{fetch(:stage)})" %>
 After=syslog.target network.target
@@ -12,7 +10,7 @@ ExecReload=/bin/kill -TSTP $MAINPID
 ExecStop=/bin/kill -TERM $MAINPID
 <%="StandardOutput=append:#{fetch(:sidekiq_log)}" if fetch(:sidekiq_log) %>
 <%="StandardError=append:#{fetch(:sidekiq_error_log)}" if fetch(:sidekiq_error_log) %>
-<%="User=#{fetch(:sidekiq_user)}" if fetch(:sidekiq_user) %>
+<%="User=#{fetch(:sidekiq_user)}" if (fetch(:sidekiq_user) && (fetch(:puma_systemctl_user) == :system)) %>
 <%="EnvironmentFile=#{fetch(:sidekiq_service_unit_env_file)}" if fetch(:sidekiq_service_unit_env_file) %>
 <% fetch(:sidekiq_service_unit_env_vars, []).each do |environment_variable| %>
   <%="Environment=#{environment_variable}" %>
@@ -27,4 +25,6 @@ Restart=on-failure
 SyslogIdentifier=sidekiq_<%= fetch(:application) %>_<%= fetch(:stage) %>
 
 [Install]
-WantedBy=default.target
+WantedBy=<%=(fetch(:sidekiq_systemctl_user) == :system) ? "multi-user.target" : "default.target"%>
+
+<% # Adapted from: https://github.com/seuros/capistrano-sidekiq/blob/master/lib/generators/capistrano/sidekiq/systemd/templates/sidekiq.service.capistrano.erb %>

lib/generators/capistrano/reliably_deploying_rails/templates/sidekiq.service.capistrano.erb

@@ -1,4 +1,4 @@
 check process <%= fetch(:sidekiq_service_unit_name) %> matching "sidekiq.*<%= fetch(:full_app_name) %>"  
-  start program = "/usr/bin/systemctl start <%= fetch(:sidekiq_service_unit_name) %>"
-  stop program = "/usr/bin/systemctl stop <%= fetch(:sidekiq_service_unit_name) %>"
+  start program = "/bin/bash -c 'XDG_RUNTIME_DIR=/run/user/$(id -u) /usr/bin/systemctl start --user <%= fetch(:sidekiq_service_unit_name) %>'" as uid "<%= fetch(:sidekiq_systemctl_user) %>" and gid "<%= fetch(:sidekiq_systemctl_user) %>"
+  stop program = "/bin/bash -c 'XDG_RUNTIME_DIR=/run/user/$(id -u) /usr/bin/systemctl stop --user <%= fetch(:sidekiq_service_unit_name) %>'" as uid "<%= fetch(:sidekiq_systemctl_user) %>" and gid "<%= fetch(:sidekiq_systemctl_user) %>"
   group <%= fetch(:sidekiq_monit_group) || fetch(:full_app_name) %>-sidekiq