[Waiting on #174] WIP - Add rake task to sync items with Amazon

.rubocop.yml

@@ -22,3 +22,7 @@ Style/NumericLiterals:
 Style/BlockDelimiters:
   Exclude:
     - spec/**/*
+
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - spec/**/*

README.md

@@ -175,7 +175,10 @@ account or login locally:
 
 By the end of this section, you should be able to search Amazon products and
 add items to wishlists on your local machine (when your logged-in user is an
-admin or site manager).
+admin or site manager). *Note: if you're adding a new API endpoint, read more
+[here][API client README].*
+
+[API client README]: lib/amazon_product_api/README.md
 
 **This step is only required for site managers and admins searching/adding
 Amazon products.** If your issue doesn't involve working with the Amazon

app/jobs/item_sync_job.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'amazon_product_api'
+
+# This job is responsible for syncing items with Amazon.
+#
+# Amazon changes prices, details, etc. pretty frequently, so this job will
+# query the Amazon record associated with the ASIN and make any required local
+# updates.
+#
+# Bang methods (ex. `#sync_batch!`) perform an HTTP request.
+#
+class ItemSyncJob < ApplicationJob
+  # Maximum number of ASINs in one request
+  BATCH_SIZE = AmazonProductAPI::ItemLookupEndpoint::ASIN_LIMIT
+
+  queue_as :default
+
+  def initialize
+    @client = AmazonProductAPI::HTTPClient.new
+  end
+
+  # Syncs all items and writes the results to the log
+  def perform(*_args)
+    Rails.logger.info bold_green('Syncing all items')
+    sync_all!
+    Rails.logger.info bold_green('Done syncing!')
+  end
+
+  private
+
+  attr_reader :client
+
+  # Syncs all database items with their Amazon sources
+  def sync_all!
+    # This is done in slices and batches to avoid Amazon rate limits
+    Item.all.each_slice(BATCH_SIZE * 3) do |batches|
+      batches.each_slice(BATCH_SIZE) { |batch| sync_batch! batch }
+      sleep 2.seconds unless Rails.env.test?
+    end
+  end
+
+  # Syncs one batch of items with Amazon (up to the batch size limit)
+  def sync_batch!(items)
+    count = items.count
+    validate_batch_size(count)
+
+    Rails.logger.info "Fetching #{count} items: #{items.map(&:asin).join(',')}"
+
+    items_updates = get_updates_for! items
+    items_updates.map { |item, updates| update_item(item, updates) }
+  end
+
+  # Returns pairs of items and their corresponding update hashes
+  def get_updates_for!(items)
+    query = client.item_lookup(*items.map(&:asin))
+    amazon_items = query.response
+    updates = amazon_items.map(&:update_hash)
+
+    items.zip(updates)
+  end
+
+  def validate_batch_size(count)
+    return unless count > BATCH_SIZE
+    raise ArgumentError,
+          "Batch size too large: #{count}/#{BATCH_SIZE}"
+  end
+
+  def update_item(item, update_hash)
+    Rails.logger.info green(
+      "Syncing item #{item.id}: (#{item.asin}) #{item.name.truncate(64)}"
+    )
+    item.assign_attributes(update_hash)
+    return unless item.changed?
+
+    Rails.logger.info bold_green("Changed:\n") + item.changes.pretty_inspect
+    item.save!
+  end
+
+  # Some styles for log text. This is so minor that it's not worth
+  # bringing in a full library.
+
+  def bold_green(text)
+    bold(green(text))
+  end
+
+  def green(text)
+    "\e[32m#{text}\e[0m"
+  end
+
+  def bold(text)
+    "\e[1m#{text}\e[22m"
+  end
+end

lib/amazon_product_api/README.md

@@ -0,0 +1,68 @@
+# Amazon Product Advertising API Client
+
+This folder contains the wrapper code for the Amazon Product Advertising API.
+For more details on the API, check the [Amazon documentation].
+
+[Amazon documentation]: http://docs.aws.amazon.com/AWSECommerceService/latest/DG/Welcome.html
+
+## Adding an Endpoint
+
+All endpoints should be subclassed from `AmazonProductAPI::Endpoint`. In order
+to add a new endpoint, you'll need to modify the template below in a few ways:
+
+  * Prepend any new attributes to the `#initialize` parameter list. Any
+    validations or processing can be done in `#initialize` as shown. Note:
+    setting `aws_credentials` is **required**!
+
+  * Add endpoint-specific request parameters to `#request_params`. These can
+    be found in the Amazon API documentation.
+
+  * Add any post-processing of the API response in `#process_response`.
+
+  * Update the class name and comments.
+
+### Endpoint Template
+
+```ruby
+require 'amazon_product_api/endpoint'
+
+module AmazonProductAPI
+  # Responsible for building and executing <...>
+  #
+  # <endpoint url>
+  #
+  # Contains all specialization logic for this endpoint including request
+  # parameters, parameter validation, and response parsing.
+  class TemplateEndpoint < Endpoint
+    # Add any parameters you need for the specific endpoint.
+    #
+    # Make sure you set `@aws_credentials`-the query won't work without it!
+    def initialize(aws_credentials)
+      # Attribute validations
+      # raise InvalidQueryError, 'reason' if ...
+
+      # Initialize attributes
+      @aws_credentials = aws_credentials
+    end
+
+    private
+
+    attr_accessor :aws_credentials # any other attrs
+
+    # Add any post-processing of the response hash.
+    def process_response(response_hash)
+      ExampleResponse.new(response_hash).item
+    end
+
+    # Include request parameters unique to this endpoint.
+    def request_params
+      {
+        # 'Operation' => 'ItemLookup',
+        # 'IdType' => 'ASIN',
+        # 'ItemId' => 'the item asin',
+        # ...
+      }
+    end
+  end
+end
+```

lib/amazon_product_api/endpoint.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module AmazonProductAPI
+  # Base representation of all Amazon Product Advertising API endpoints.
+  #
+  # http://docs.aws.amazon.com/AWSECommerceService/latest/DG/\
+  #   CHAP_OperationListAlphabetical.html
+  #
+  # Any general logic relating to lookup, building the query string,
+  # authentication signatures, etc. should live in this class. Specializations
+  # (including specific request parameters and response parsing) should live in
+  # endpoint subclasses.
+  class Endpoint
+    require 'httparty'
+    require 'time'
+    require 'uri'
+    require 'openssl'
+    require 'base64'
+
+    # The region you are interested in
+    ENDPOINT    = 'webservices.amazon.com'
+    REQUEST_URI = '/onca/xml'
+
+    # Generates the signed URL
+    def url
+      raise InvalidQueryError, 'Missing AWS credentials' unless aws_credentials
+
+      "http://#{ENDPOINT}#{REQUEST_URI}" +    # base
+        "?#{canonical_query_string}" +        # query
+        "&Signature=#{uri_escape(signature)}" # signature
+    end
+
+    # Sends the HTTP request
+    def get(http: HTTParty)
+      http.get(url)
+    end
+
+    # Performs the search query and returns the processed response
+    def response(http: HTTParty, logger: Rails.logger)
+      response = parse_response get(http: http)
+      process_response(response)
+    end
+
+    private
+
+    attr_reader :aws_credentials
+
+    # Takes the response hash and returns the processed API response
+    #
+    # This must be implemented for each individual endpoint.
+    def process_response(_response_hash)
+      raise NotImplementedError, 'Implement this method in your subclass.'
+    end
+
+    # Returns a hash of request parameters unique to the endpoint
+    #
+    # This must be implemented for each individual endpoint.
+    def request_params
+      raise NotImplementedError, 'Implement this method in your subclass.'
+    end
+
+    def params
+      params = request_params.merge(
+        'Service'         => 'AWSECommerceService',
+        'AWSAccessKeyId'  => aws_credentials.access_key,
+        'AssociateTag'    => aws_credentials.associate_tag
+      )
+
+      # Set current timestamp if not set
+      params['Timestamp'] ||= Time.now.gmtime.iso8601
+      params
+    end
+
+    def parse_response(response)
+      Hash.from_xml(response.body)
+    end
+
+    # Generates the signature required by the Product Advertising API
+    def signature
+      Base64.encode64(digest_with_key(string_to_sign)).strip
+    end
+
+    def string_to_sign
+      "GET\n#{ENDPOINT}\n#{REQUEST_URI}\n#{canonical_query_string}"
+    end
+
+    def canonical_query_string
+      params.sort
+            .map { |key, value| "#{uri_escape(key)}=#{uri_escape(value)}" }
+            .join('&')
+    end
+
+    def digest_with_key(string)
+      OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'),
+                           aws_credentials.secret_key,
+                           string)
+    end
+
+    def uri_escape(phrase)
+      CGI.escape(phrase.to_s)
+    end
+  end
+end

lib/amazon_product_api/http_client.rb

@@ -19,8 +19,8 @@ def item_search(query:, page: 1)
       ItemSearchEndpoint.new(query, page, aws_credentials)
     end
 
-    def item_lookup(asin)
-      ItemLookupEndpoint.new(asin, aws_credentials)
+    def item_lookup(*asin)
+      ItemLookupEndpoint.new(*asin, aws_credentials)
     end
 
     private