Merge pull request #183 from MITLibraries/libkey

Add LibKey API option for DOI and PMID metadata

Author: JPrevost

.env.test

@@ -1,3 +1,5 @@
 DETECTOR_VERSION=1
 LINKRESOLVER_BASEURL=https://mit.primo.exlibrisgroup.com/discovery/openurl?institution=01MIT_INST&rfr_id=info:sid/mit.tacos.api&vid=01MIT_INST:MIT
 TACOS_EMAIL=[email protected]
+LIBKEY_KEY=FAKE_LIBKEY_KEY
+LIBKEY_ID=FAKE_LIBKEY_ID

README.md

@@ -69,6 +69,11 @@ changes, this is the signal which indicates that terms need to be re-evaluated.
 
 ### Optional
 
+`LIBKEY_KEY`: LibKey API key. Required if `LIBKEY_DOI` or `LIBKEY_PMID` are set.
+`LIBKEY_ID`: LibKey Library ID. Required if `LIBKEY_DOI` or `LIBKEY_PMID` are set.
+`LIBKEY_DOI`: If set, use LibKey for DOI metadata lookups. If not set, Unpaywall is used.
+`LIBKEY_PMID`: If set, use LibKey for PMID metadata lookups. If not set, NCBI Entrez is used.
+
 `PLATFORM_NAME`: The value set is added to the header after the MIT Libraries logo. The logic and CSS for this comes
 from our theme gem.
 

app/graphql/types/details_type.rb

@@ -7,10 +7,13 @@ class DetailsType < Types::BaseObject
     field :date, String
     field :doi, String
     field :issns, [String]
+    field :journal_image, String
+    field :journal_link, String
     field :journal_name, String
     field :link_resolver_url, String
     field :oa, Boolean
     field :oa_status, String
+    field :pmid, String
     field :publisher, String
     field :title, String
 
@@ -19,7 +22,7 @@ def issns
     end
 
     def authors
-      @object[:authors]&.split(',')
+      @object[:authors]&.split(';')
     end
   end
 end

app/graphql/types/standard_identifiers_type.rb

@@ -15,12 +15,30 @@ def details
       when :barcode
         LookupBarcode.new.info(@object[:value])
       when :doi
-        LookupDoi.new.info(@object[:value])
+        doi
       when :isbn
         LookupIsbn.new.info(@object[:value])
       when :issn
         LookupIssn.new.info(@object[:value])
       when :pmid
+        pmid
+      end
+    end
+
+    # doi handles determining which doi lookup method to use
+    def doi
+      if ENV.fetch('LIBKEY_DOI', false)
+        LookupLibkey.info(doi: @object[:value])
+      else
+        LookupDoi.new.info(@object[:value])
+      end
+    end
+
+    # pmid handles determining which pmid lookup method to use
+    def pmid
+      if ENV.fetch('LIBKEY_PMID', false)
+        LookupLibkey.info(pmid: @object[:value])
+      else
         LookupPmid.new.info(@object[:value].split.last)
       end
     end

app/models/detector/standard_identifiers.rb

@@ -26,6 +26,7 @@ def initialize(phrase)
       pattern_checker(phrase)
       strip_invalid_issns
       strip_invalid_isbns
+      strip_pmid_prefix
     end
 
     # The record method will consult the set of regex-based detectors that are defined in
@@ -63,6 +64,12 @@ def patterns
       }
     end
 
+    # strip_pmid_prefix removes the PMID:/pmid: prefix from the detected value. The regex needs that, but the
+    # actual value of the identifier should not include those prefixes.
+    def strip_pmid_prefix
+      @detections[:pmid] = @detections[:pmid].gsub(/pmid:|PMID:/, '').strip if @detections[:pmid].present?
+    end
+
     # strip_invalid_isbns coordinates the logic to remove ISBNs that are not valid from our list of detected ISBNs
     #
     # ISBNs cannot be validated via regex. Regex gives us a list of candidates that look like ISBNs. We remove invalid

app/models/lookup_barcode.rb

@@ -37,10 +37,18 @@ def extract_metadata(xml)
       title: xml.xpath('//dc:title', 'dc' => 'http://purl.org/dc/elements/1.1/').text,
       date: xml.xpath('//dc:date', 'dc' => 'http://purl.org/dc/elements/1.1/').text,
       publisher: xml.xpath('//dc:publisher', 'dc' => 'http://purl.org/dc/elements/1.1/').text,
-      authors: xml.xpath('//dc:contributor', 'dc' => 'http://purl.org/dc/elements/1.1/').text
+      authors: authors(xml)
     }
   end
 
+  def authors(xml)
+    authors = []
+    xml.xpath('//dc:contributor', 'dc' => 'http://purl.org/dc/elements/1.1/').each do |author|
+      authors << author.text
+    end
+    authors.join(';')
+  end
+
   def url(barcode)
     "https://mit.alma.exlibrisgroup.com/view/sru/01MIT_INST?version=1.2&operation=searchRetrieve&recordSchema=dc&query=alma.all_for_ui=#{barcode}"
   end

app/models/lookup_isbn.rb

@@ -32,7 +32,7 @@ def fetch_authors(isbn_json)
       json = parse_response(url)
       json['name']
     end
-    author_names.join(' ; ')
+    author_names.join(';')
   end
 
   def parse_response(url)

app/models/lookup_libkey.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# LookupLibkey can take a DOI or PMID and return metadata, link resolver links, and journal browse links.
+class LookupLibkey
+  BASEURL = 'https://public-api.thirdiron.com/public/v1/libraries'
+
+  # Info is the main entry point into the LookupLibkey Class.
+  #
+  # @param doi [String]
+  # @param pmid [String]
+  # @return [Hash] or nil
+  def self.info(doi: nil, pmid: nil)
+    return unless expected_env?
+
+    if doi.present?
+      external_url = construct_url(doi:)
+      Rails.logger.debug(external_url)
+      external_data = fetch(external_url)
+      return if external_data == 'Error'
+
+      extract_metadata(external_data)
+    elsif pmid.present?
+      external_url = construct_url(pmid:)
+      Rails.logger.debug(external_url)
+
+      external_data = fetch(external_url)
+      return if external_data == 'Error'
+
+      extract_metadata(external_data)
+    else
+      Rails.logger.error('No doi or pmid provided to LookupLibkey')
+      nil
+    end
+  end
+
+  # expected_env? confirms both required variables are set
+  #
+  # @return Boolean
+  def self.expected_env?
+    Rails.logger.error('No LIBKEY_KEY set') if libkey_key.nil?
+
+    Rails.logger.error('No LIBKEY_ID set') if libkey_id.nil?
+
+    libkey_id.present? && libkey_key.present?
+  end
+
+  # using method instead of constant to allow for mutating in testing without causing sporadic failures
+  def self.libkey_key
+    ENV.fetch('LIBKEY_KEY', nil)
+  end
+
+  # using method instead of constant to allow for mutating in testing without causing sporadic failures
+  def self.libkey_id
+    ENV.fetch('LIBKEY_ID', nil)
+  end
+
+  # extract_metadata maps data from the LibKey response to an internal hash
+  #
+  # @return Hash
+  def self.extract_metadata(external_data)
+    {
+      title: external_data['data']['title'],
+      authors: external_data['data']['authors'].gsub('; ', ';'),
+      doi: external_data['data']['doi'],
+      pmid: external_data['data']['pmid'],
+      oa: external_data['data']['openAccess'],
+      date: external_data['data']['date'],
+      journal_name: external_data['included'].first['title'],
+      journal_issns: external_data['included'].first['issn'],
+      journal_image: external_data['included'].first['coverImageUrl'],
+      journal_link: external_data['included'].first['browzineWebLink'],
+      link_resolver_url: external_data['data']['bestIntegratorLink']['bestLink']
+    }
+  end
+
+  # https://thirdiron.atlassian.net/wiki/spaces/BrowZineAPIDocs/pages/65929220/BrowZine+Public+API+Overview
+  # https://thirdiron.atlassian.net/wiki/spaces/BrowZineAPIDocs/pages/65699928/Article+DOI+PMID+Lookup+Endpoint+LibKey
+  # public/v1/libraries/:library_id/articles/doi/:article_doi?access_token=ffffffff-ffff-ffff-ffff-ffffffffffff
+  # /public/v1/libraries/:library_id/articles/pmid/:article_pmid?access_token=ffffffff-ffff-ffff-ffff-ffffffffffff
+  def self.construct_url(doi: nil, pmid: nil)
+    if doi.present?
+      "#{BASEURL}/#{libkey_id}/articles/doi/#{doi}?include=journal&access_token=#{libkey_key}"
+    elsif pmid.present?
+      "#{BASEURL}/#{libkey_id}/articles/pmid/#{pmid}?include=journal&access_token=#{libkey_key}"
+    else
+      Rails.logger.error('No PMID or DOI provided to LookupLibkey.url()')
+      nil
+    end
+  end
+
+  # Fetch performs the HTTP calls, parses JSON for successful requests.
+  def self.fetch(url)
+    resp = HTTP.headers(accept: 'application/json').get(url)
+    if resp.status == 200
+      JSON.parse(resp.to_s)
+    else
+      Rails.logger.debug do
+        'Fact lookup error. DOI or PMID detected but LibKey returned no data or otherwise errored'
+      end
+      Rails.logger.debug { "Response status: #{resp.status}" }
+      Rails.logger.debug { "URL: #{url}" }
+      'Error'
+    end
+  end
+end

test/models/detector/standard_identifiers_test.rb

@@ -195,7 +195,7 @@ class StandardIdentifiersTest < ActiveSupport::TestCase
     test 'pmid detected in string' do
       actual = Detector::StandardIdentifiers.new('Citation and stuff PMID: 35648703 more stuff.').detections
 
-      assert_equal('PMID: 35648703', actual[:pmid])
+      assert_equal('35648703', actual[:pmid])
     end
 
     test 'pmid examples' do
@@ -204,7 +204,7 @@ class StandardIdentifiersTest < ActiveSupport::TestCase
       samples.each do |pmid|
         actual = Detector::StandardIdentifiers.new(pmid).detections
 
-        assert_equal(pmid, actual[:pmid])
+        assert_equal(pmid.gsub(/PMID:|pmid:/, '').strip, actual[:pmid])
       end
     end
 

test/models/lookup_libkey_test.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class LookupLibkeyTest < ActiveSupport::TestCase
+  test 'metadata object is returned with expected fields for dois' do
+    VCR.use_cassette('libkey doi 10.1038/d41586-023-03497-2') do
+      metadata = LookupLibkey.info(doi: '10.1038/d41586-023-03497-2')
+
+      expected_keys = %i[title authors doi pmid oa date journal_name journal_issns journal_image journal_link
+                         link_resolver_url]
+
+      expected_keys.each do |key|
+        assert_includes(metadata.keys, key)
+      end
+    end
+  end
+
+  test 'metadata object is returned with expected fields for pmids' do
+    VCR.use_cassette('libkey pmid 10490598') do
+      metadata = LookupLibkey.info(pmid: '10490598')
+
+      expected_keys = %i[title authors doi pmid oa date journal_name journal_issns journal_image journal_link
+                         link_resolver_url]
+
+      expected_keys.each do |key|
+        assert_includes(metadata.keys, key)
+      end
+    end
+  end
+
+  test 'link resolver url returns expected value for dois' do
+    VCR.use_cassette('libkey doi 10.1038/d41586-023-03497-2') do
+      metadata = LookupLibkey.info(doi: '10.1038/d41586-023-03497-2')
+
+      expected_url = 'https://libkey.io/libraries/FAKE_LIBKEY_ID/articles/594388926/full-text-file?utm_source=api_3735'
+
+      assert_equal(expected_url, metadata[:link_resolver_url])
+    end
+  end
+
+  test 'link resolver url returns expected value for pmids' do
+    VCR.use_cassette('libkey pmid 10490598') do
+      metadata = LookupLibkey.info(pmid: '10490598')
+
+      expected_url = 'https://libkey.io/libraries/FAKE_LIBKEY_ID/articles/56753128/full-text-file?utm_source=api_3735'
+
+      assert_equal(expected_url, metadata[:link_resolver_url])
+    end
+  end
+
+  test 'doi or pmid are required' do
+    error = assert_raises(ArgumentError) do
+      LookupLibkey.info('no pmid or doi argument name given')
+    end
+    assert_equal 'wrong number of arguments (given 1, expected 0)', error.message
+  end
+
+  test 'LIBKEY_KEY is required' do
+    ClimateControl.modify LIBKEY_KEY: nil do
+      assert_nil(LookupLibkey.info(doi: '10.1038/d41586-023-03497-2'))
+    end
+  end
+
+  test 'LIBKEY_ID is required' do
+    ClimateControl.modify LIBKEY_ID: nil do
+      assert_nil(LookupLibkey.info(doi: '10.1038/d41586-023-03497-2'))
+    end
+  end
+
+  test 'blank doi and blank pmid returns nil' do
+    metadata = LookupLibkey.info(doi: '')
+
+    assert_nil(metadata)
+
+    metadata = LookupLibkey.info(pmid: '')
+
+    assert_nil(metadata)
+
+    metadata = LookupLibkey.info(pmid: '', doi: '')
+
+    assert_nil(metadata)
+  end
+
+  test 'contstruct url for doi' do
+    expected_url = 'https://public-api.thirdiron.com/public/v1/libraries/FAKE_LIBKEY_ID/articles/doi/my_doi?include=journal&access_token=FAKE_LIBKEY_KEY'
+    actual_url = LookupLibkey.construct_url(doi: 'my_doi')
+
+    assert_equal(expected_url, actual_url)
+  end
+
+  test 'contstruct url for pmid' do
+    expected_url = 'https://public-api.thirdiron.com/public/v1/libraries/FAKE_LIBKEY_ID/articles/pmid/my_pmid?include=journal&access_token=FAKE_LIBKEY_KEY'
+    actual_url = LookupLibkey.construct_url(pmid: 'my_pmid')
+
+    assert_equal(expected_url, actual_url)
+  end
+
+  test 'construct url with no pmid or do returns nil' do
+    actual_url = LookupLibkey.construct_url
+
+    assert_nil(actual_url)
+  end
+
+  test 'invalid doi lookup returns nil' do
+    VCR.use_cassette('libkey doi nonsense') do
+      metadata = LookupLibkey.info(doi: 'nonsense')
+
+      assert_nil(metadata)
+    end
+  end
+
+  test 'invalid pmid lookup returns nil' do
+    VCR.use_cassette('libkey pmid nonsense') do
+      metadata = LookupLibkey.info(pmid: 'nonsense')
+
+      assert_nil(metadata)
+    end
+  end
+end

test/test_helper.rb

@@ -23,6 +23,15 @@
     ENV.fetch('TACOS_EMAIL', nil).to_s
   end
 
+  # Filter Libkey Key
+  config.filter_sensitive_data('FAKE_LIBKEY_KEY') do
+    ENV.fetch('LIBKEY_KEY', nil).to_s
+  end
+  # Filter LibKey ID
+  config.filter_sensitive_data('FAKE_LIBKEY_ID') do
+    ENV.fetch('LIBKEY_ID', nil).to_s
+  end
+
   config.before_record do |interaction|
     header = interaction.response&.headers&.[]('Report-To')
     header&.each do |redacted_text|