Configure sqs connection and initial event_import_worker

Author: wendelfabianchinsamy

Gemfile.lock

@@ -393,4 +393,4 @@ RUBY VERSION
    ruby 3.1.6p260
 
 BUNDLED WITH
-   2.5.6
+   2.5.6

README.md

@@ -1,3 +1,18 @@
 # DataCite Events Service
 
-DataCite Events Service
+This is the code repository for the DataCite Events REST API.
+
+### Adding new Shoryuken workers
+
+1. Add workers to the app/workers directory
+2. Ensure you set the shoryuken_options e.g. `shoryuken_options queue: -> { "#{ENV['RAILS_ENV']}\_events" }, auto_delete: true`
+3. Queues use environment prefixes. The prefix is set with the environment variable RAILS_ENV locally.
+4. Add a requires statement for your queue in config/initializers/\_shoryuken.rb file e.g. `require Rails.root.join("app/workers/event_import_worker.rb")`
+
+### Starting the Shoryuken workers
+
+1. Workers are disabled in development by default.
+2. The environment variable DISABLE_QUEUE_WORKER is used in development to switch the worker on or off when you start the container.
+3. The DISABLE_QUEUE_WORKER is set to nil by defalt in the docker-compose.yml i.e. DISABLE_QUEUE_WORKER=
+4. If you want to start workers by default when the container spins up set DISABLE_QUEUE_WORKER to a truthy value i.e. 1, TRUE, true, foobar
+5. Alternatively you can start the workers manually by bashing into the events_api container and running `bundle exec shoryuken -R -C config/shoryuken.yml`

app/workers/event_import_worker.rb

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
-# The EventImportWorker subscribes to the events queue (with environment prefix i.e. stage_events production_events).
-# It deqeues a message from the events queue.
-# Determines if the event exists:
-#   If the event does not exist, we create a new event (mysql and opensearch)
-#   If the event does exist, we update the existing event (mysql and opensearch)
 class EventImportWorker
   include Shoryuken::Worker
 
-  # Ensures that we use the rails environment on our queue prefix
-  shoryuken_options queue: -> { "#{ENV["RAILS_ENV"]}_events" }, auto_delete: true
+  shoryuken_options queue: -> { "#{ENV['RAILS_ENV']}_events" }, auto_delete: true
 
-  def perform(sqs_message = nil, data = nil)
+  def perform(sqs_message=nil, data=nil)
     log_prefix = "[Events:EventImportWorker]"
+
+    if data.blank?
+      Rails.logger.info("#{log_prefix} Message data was blank")
+      return
+    end
+
     log_identifier = "subj_id: #{data["subjId"]}, " \
       "obj_id: #{data["objId"]}, " \
       "source_id: #{data["sourceId"]}, " \
@@ -22,22 +22,18 @@ def perform(sqs_message = nil, data = nil)
     Rails.logger.info("#{log_prefix} Searching for event with #{log_identifier}")
 
     event = Event.find_by(
-      sub_id: data["subjId"],
+      subj_id: data["subjId"],
       obj_id: data["objId"],
       source_id: data["sourceId"],
       relation_type_id: data["relationTypeId"],
     )
 
     if event.blank?
-      Rails.logger.info("#{LOG_PREFIX} Creating a new event with #{log_identifier}")
-      # create the event in mysql
-      # and index the event
+      Rails.logger.info("#{log_prefix} Creating a new event with #{log_identifier}")
     else
-      Rails.logger.info("#{LOG_PREFIX} Update an existing event with #{log_identifier}")
-      # update the event in mysql
-      # and index the event
+      Rails.logger.info("#{log_prefix} Update an existing event with #{log_identifier}")
     end
 
-    Rails.logger.info("#{LOG_PREFIX} Update existing event with #{log_identifier}")
+    Rails.logger.info("#{log_prefix} End of event message processing for #{log_identifier}")
   end
 end

config/application.rb

@@ -30,12 +30,12 @@ class Application < Rails::Application
     # Autoload paths
     # You typically add directories to autoload_paths so that classes and modules are loaded
     # automatically as they are referenced. This makes development more convenient
-    # config.autoload_paths << "#{config.root}/app/models/factories"
+    # config.autoload_paths << "#{config.root}/app/workers"
 
     # Eager load paths
     # You typically add directories to eager_load_paths to ensure that all necessary classes
     # and modules are loaded at startup, improving performance in production.
-    # config.eager_load_paths << "#{config.root}/app/models/factories"
+    # config.eager_load_paths << "#{config.root}/app/workers"
 
     config.api_only = true
 

config/initializers/_shoryuken.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
+require Rails.root.join("app/workers/event_import_worker.rb")
+
 if Rails.env.development?
   Aws.config.update({
     endpoint: ENV["AWS_ENDPOINT"],
     region: ENV["AWS_REGION"],
-    credentials: Aws::Credentials.new(ENV["AWS_ACCESS_KEY"], ENV["AWS_SECRET_ACCESS_KEY"]),
+    credentials: Aws::Credentials.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_ACCESS_KEY"]),
   })
 end
 
+# This ensures that we work with queues that
+# have an environment prefix i.e. stage_events or development_events
 Shoryuken.active_job_queue_name_prefixing = true

docker-compose.yml

@@ -2,6 +2,7 @@ services:
   api:
     container_name: events_api
     environment:
+      - RAILS_ENV=development
       - LOG_LEVEL=debug
       - MYSQL_USER=root
       - MYSQL_PASSWORD=
@@ -14,9 +15,9 @@ services:
       - ES_SCHEME=http
       - ELASTIC_PASSWORD=AnUnsecurePassword123
       - AWS_ENDPOINT=http://localstack:4566
-      - AWS_ACCESS_KEY=test
-      - AWS_SECRET_ACCESS_KEY=test
       - AWS_REGION=us-east-1
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
       - DISABLE_QUEUE_WORKER=
     build:
       context: ./

vendor/docker/shoryuken.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 cd /home/app/webapp
 exec 2>&1
-# If AWS_REGION is set and not explicitly disabled with DISABLE_QUEUE_WORKER, start shoryuken
+# Disabled with DISABLE_QUEUE_WORKER, start shoryuken
 if [ -n "$DISABLE_QUEUE_WORKER" ]; then
   exec /sbin/setuser app bundle exec shoryuken -R -C config/shoryuken.yml
 fi