Merge pull request #17 from datacite/configure-sqs-connection

Configure sqs connection

Author: wendelfabianchinsamy

Gemfile.lock

@@ -393,4 +393,4 @@ RUBY VERSION
    ruby 3.1.6p260
 
 BUNDLED WITH
-   2.5.6
+   2.5.6

README.md

@@ -1,3 +1,18 @@
 # DataCite Events Service
 
-DataCite Events Service
+This is the code repository for the DataCite Events REST API.
+
+### Adding new Shoryuken workers
+
+1. Add workers to the app/workers directory
+2. Ensure you set the shoryuken_options e.g. `shoryuken_options queue: -> { "#{ENV['RAILS_ENV']}\_events" }, auto_delete: true`
+3. Queues use environment prefixes. The prefix is set with the environment variable RAILS_ENV locally.
+4. Add a requires statement for your queue in config/initializers/\_shoryuken.rb file e.g. `require Rails.root.join("app/workers/event_import_worker.rb")`
+
+### Starting the Shoryuken workers
+
+1. Workers are disabled in development by default.
+2. The environment variable DISABLE_QUEUE_WORKER is used in development to switch the worker on or off when you start the container.
+3. The DISABLE_QUEUE_WORKER is set to nil by defalt in the docker-compose.yml i.e. DISABLE_QUEUE_WORKER=
+4. If you want to start workers by default when the container spins up set DISABLE_QUEUE_WORKER to a truthy value i.e. 1, TRUE, true, foobar
+5. Alternatively you can start the workers manually by bashing into the events_api container and running `bundle exec shoryuken -R -C config/shoryuken.yml`

app/workers/event_import_worker.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class EventImportWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: -> { "#{ENV["RAILS_ENV"]}_events" }, auto_delete: true
+
+  def perform(sqs_message = nil, data = nil)
+    log_prefix = "[Events:EventImportWorker]"
+
+    if data.blank?
+      Rails.logger.info("#{log_prefix} Message data was blank")
+      return
+    end
+
+    log_identifier = "subj_id: #{data["subjId"]}, " \
+      "obj_id: #{data["objId"]}, " \
+      "source_id: #{data["sourceId"]}, " \
+      "relation_type_id: #{data["relationTypeId"]}"
+
+    Rails.logger.info("#{log_prefix} Start of event message processing for #{log_identifier}")
+    Rails.logger.info("#{log_prefix} Searching for event with #{log_identifier}")
+
+    event = Event.find_by(
+      subj_id: data["subjId"],
+      obj_id: data["objId"],
+      source_id: data["sourceId"],
+      relation_type_id: data["relationTypeId"],
+    )
+
+    if event.blank?
+      Rails.logger.info("#{log_prefix} Creating a new event with #{log_identifier}")
+    else
+      Rails.logger.info("#{log_prefix} Update an existing event with #{log_identifier}")
+    end
+
+    Rails.logger.info("#{log_prefix} End of event message processing for #{log_identifier}")
+  end
+end

config/application.rb

@@ -30,12 +30,12 @@ class Application < Rails::Application
     # Autoload paths
     # You typically add directories to autoload_paths so that classes and modules are loaded
     # automatically as they are referenced. This makes development more convenient
-    # config.autoload_paths << "#{config.root}/app/models/factories"
+    # config.autoload_paths << "#{config.root}/app/workers"
 
     # Eager load paths
     # You typically add directories to eager_load_paths to ensure that all necessary classes
     # and modules are loaded at startup, improving performance in production.
-    # config.eager_load_paths << "#{config.root}/app/models/factories"
+    # config.eager_load_paths << "#{config.root}/app/workers"
 
     config.api_only = true
 

config/initializers/_shoryuken.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
+require Rails.root.join("app/workers/event_import_worker.rb")
+
 if Rails.env.development?
   Aws.config.update({
     endpoint: ENV["AWS_ENDPOINT"],
-    region: "us-east-1",
-    credentials: Aws::Credentials.new("test", "test"),
+    region: ENV["AWS_REGION"],
+    credentials: Aws::Credentials.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_ACCESS_KEY"]),
   })
 end
 
+# This ensures that we work with queues that
+# have an environment prefix i.e. stage_events or development_events
 Shoryuken.active_job_queue_name_prefixing = true

docker-compose.yml

@@ -2,6 +2,7 @@ services:
   api:
     container_name: events_api
     environment:
+      - RAILS_ENV=development
       - LOG_LEVEL=debug
       - MYSQL_USER=root
       - MYSQL_PASSWORD=
@@ -13,6 +14,11 @@ services:
       - ES_PORT=9200
       - ES_SCHEME=http
       - ELASTIC_PASSWORD=AnUnsecurePassword123
+      - AWS_ENDPOINT=http://localstack:4566
+      - AWS_REGION=us-east-1
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+      - DISABLE_QUEUE_WORKER=
     build:
       context: ./
       dockerfile: Dockerfile

vendor/docker/shoryuken.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 cd /home/app/webapp
 exec 2>&1
-# If AWS_REGION is set and not explicitly disabled with DISABLE_QUEUE_WORKER, start shoryuken
-if [ -n "$AWS_REGION" ]; then
+# Disabled with DISABLE_QUEUE_WORKER, start shoryuken
+if [ -n "$DISABLE_QUEUE_WORKER" ]; then
   exec /sbin/setuser app bundle exec shoryuken -R -C config/shoryuken.yml
 fi