Merge pull request #1437 from datacite/funded-by

Adds funded-by and include-funder-child-organizations URL parameters

Author: codycooperross

app/controllers/datacite_dois_controller.rb

@@ -140,6 +140,8 @@ def index
           sort: sort,
           random: params[:random],
           client_type: params[:client_type],
+          funded_by: params[:funded_by],
+          include_funder_child_organizations: params[:include_funder_child_organizations],
         )
     end
 
@@ -333,6 +335,8 @@ def index
                       composite: params[:composite],
                       affiliation: params[:affiliation],
                       publisher: params[:publisher],
+                      funded_by: params[:funded_by],
+                      include_funder_child_organizations: params[:include_funder_child_organizations],
                       # The cursor link should be an array of values, but we want to encode it into a single string for the URL
                       "page[cursor]" =>
                         page[:cursor] ? make_cursor(results) : nil,

app/models/concerns/rorable.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Rorable
+  extend ActiveSupport::Concern
+
+  def get_ror_from_crossref_funder_id(funder_id)
+    funder_id_suffix = funder_id.split("10.13039/").last
+    FUNDER_TO_ROR[funder_id_suffix]
+  end
+
+  def get_ror_parents(ror_id)
+    normalized_ror = "https://#{ror_from_url(ror_id)}"
+    ROR_HIERARCHY[normalized_ror]&.fetch("ancestors", []) || []
+  end
+end

app/models/doi.rb

@@ -14,6 +14,7 @@ class Doi < ApplicationRecord
   include Metadatable
   include Cacheable
   include Dateable
+  include Rorable
 
   # include helper module for generating random DOI suffixes
   include Helpable
@@ -269,6 +270,8 @@ def validate_publisher_obj?(doi)
       indexes :organization_id,                type: :keyword
       indexes :fair_organization_id,           type: :keyword
       indexes :related_dmp_organization_id,    type: :keyword
+      indexes :funder_rors,                    type: :keyword
+      indexes :funder_parent_rors,           type: :keyword
       indexes :client_id_and_name,             type: :keyword
       indexes :provider_id_and_name,           type: :keyword
       indexes :resource_type_id_and_name,      type: :keyword
@@ -639,6 +642,8 @@ def as_indexed_json(_options = {})
       "organization_id" => organization_id,
       "fair_organization_id" => fair_organization_id,
       "related_dmp_organization_id" => related_dmp_organization_and_affiliation_id,
+      "funder_rors" => funder_rors,
+      "funder_parent_rors" => funder_parent_rors,
       "affiliation_id_and_name" => affiliation_id_and_name,
       "fair_affiliation_id_and_name" => fair_affiliation_id_and_name,
       "media_ids" => media_ids,
@@ -1172,6 +1177,22 @@ def self.query(query, options = {})
     filter << { term: { "client.client_type" =>  options[:client_type] } } if options[:client_type]
     filter << { term: { "types.resourceTypeGeneral" => "PhysicalObject" } } if options[:client_type] == "igsnCatalog"
 
+    if options[:funded_by].present?
+      normalized_funder = "https://#{ror_from_url(options[:funded_by])}"
+      if options[:include_funder_child_organizations] == "true"
+        filter << {
+          bool: {
+            should: [
+              { term: { "funder_rors": normalized_funder } },
+              { term: { "funder_parent_rors": normalized_funder } }
+            ],
+            minimum_should_match: 1
+          }
+        }
+      else
+        filter << { term: { "funder_rors": normalized_funder } }
+      end
+    end
     # match either one of has_affiliation, has_organization, or has_funder
     if options[:has_organization].present?
       should << { term: { "creators.nameIdentifiers.nameIdentifierScheme" => "ROR" } }
@@ -1958,6 +1979,27 @@ def fair_affiliation_id_and_name
     end
   end
 
+  def funder_rors
+    Array.wrap(funding_references).reduce([]) do |sum, f|
+      result = if f.is_a?(Hash) && f.fetch("funderIdentifierType", nil) == "ROR" && f.fetch("funderIdentifier", nil).present?
+        ror = ror_from_url(f.fetch("funderIdentifier", nil))
+        ror.present? ? "https://#{ror}" : nil
+      elsif f.is_a?(Hash) && f.fetch("funderIdentifierType", nil) == "Crossref Funder ID" && f.fetch("funderIdentifier", nil).present?
+        get_ror_from_crossref_funder_id(f.fetch("funderIdentifier", nil))
+      end
+
+      sum << result if result.present?
+      sum
+    end.compact.uniq
+  end
+
+  def funder_parent_rors
+    Array.wrap(funder_rors).reduce([]) do |sum, ror|
+      ancestors = get_ror_parents(ror)
+      sum.concat(ancestors) if ancestors.present?
+      sum.uniq
+    end
+  end
 
   def prefix
     doi.split("/", 2).first if doi.present?

app/resources/funder_to_ror.json

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+FUNDER_TO_ROR = JSON.parse(File.read(Rails.root.join("app/resources/funder_to_ror.json"))).freeze
+ROR_HIERARCHY = JSON.parse(File.read(Rails.root.join("app/resources/ror_hierarchy.json"))).freeze

app/resources/ror_hierarchy.json

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+describe "Rorable", type: :model do
+  describe "data is loaded" do
+    it "loads Crossref Funder Id to ROR mapping" do
+      expect(FUNDER_TO_ROR).to be_a(Hash)
+      expect(FUNDER_TO_ROR).not_to be_empty
+    end
+
+    it "loads ROR hierarchy mapping" do
+      expect(ROR_HIERARCHY).to be_a(Hash)
+      expect(ROR_HIERARCHY).not_to be_empty
+    end
+  end
+
+  describe "Crossref Funder ID to ROR mapping" do
+    let(:doi) { create(:doi) }
+
+    it "maps Crossref Funder ID without https://doi.org to ROR" do
+      funder_id = "10.13039/100010552"
+      ror_id = doi.get_ror_from_crossref_funder_id(funder_id)
+      expect(ror_id).to eq("https://ror.org/04ttjf776")
+    end
+
+    it "does not map invalid Crossref Funder ID to ROR" do
+      funder_id = "10.77777/100010552"
+      ror_id = doi.get_ror_from_crossref_funder_id(funder_id)
+      expect(ror_id).to eq(nil)
+    end
+
+    it "maps Crossref Funder ID with https://doi.org to ROR" do
+      funder_id = "https://doi.org/10.13039/100010552"
+      ror_id = doi.get_ror_from_crossref_funder_id(funder_id)
+      expect(ror_id).to eq("https://ror.org/04ttjf776")
+    end
+
+    it "maps Crossref Funder ID with https://doi.org to ROR" do
+      funder_id = "https://doi.org/10.13039/100010552"
+      ror_id = doi.get_ror_from_crossref_funder_id(funder_id)
+      expect(ror_id).to eq("https://ror.org/04ttjf776")
+    end
+  end
+
+  describe "ROR to ancestor mapping" do
+    let(:doi) { create(:doi) }
+
+    it "maps ROR URL to ancestor" do
+      ror_id = "https://ror.org/00a0jsq62"
+      ancestors = doi.get_ror_parents(ror_id)
+      expect(ancestors).to eq(["https://ror.org/04cw6st05"])
+    end
+
+    it "maps incomplete ROR URL to ancestor" do
+      ror_id = "ror.org/00a0jsq62"
+      ancestors = doi.get_ror_parents(ror_id)
+      expect(ancestors).to eq(["https://ror.org/04cw6st05"])
+    end
+
+    it "maps ROR suffix to ancestor" do
+      ror_id = "00a0jsq62"
+      ancestors = doi.get_ror_parents(ror_id)
+      expect(ancestors).to eq(["https://ror.org/04cw6st05"])
+    end
+
+    it "does not map invalid ROR to ancestor" do
+      ror_id = "doi.org/00a0jsq62"
+      ancestors = doi.get_ror_parents(ror_id)
+      expect(ancestors).to eq([])
+    end
+  end
+end

config/initializers/load_ror_data.rb

@@ -2329,4 +2329,38 @@
       })
     end
   end
+
+  describe "with funding references" do
+    let(:doi) { create(:doi,
+      funding_references:
+        [
+            {
+              "awardUri": "info:eu-repo/grantAgreement/EC/FP7/282625/",
+              "awardTitle": "MOTivational strength of ecosystem services and alternative ways to express the value of BIOdiversity",
+              "funderName": "European Commission",
+              "awardNumber": "282625",
+              "funderIdentifier": "https://doi.org/10.13039/501100000780",
+              "funderIdentifierType": "Crossref Funder ID"
+            },
+            {
+              "awardUri": "info:eu-repo/grantAgreement/EC/FP7/284382/",
+              "awardTitle": "Institutionalizing global genetic-resource commons. Global Strategies for accessing and using essential public knowledge assets in the life sciences.",
+              "funderName": "European Commission",
+              "awardNumber": "284382",
+              "funderIdentifier": "https://ror.org/00a0jsq62",
+              "funderIdentifierType": "ROR"
+            }
+        ]
+      ) }
+
+    it "has normalized funding references in funder_rors" do
+      expect(doi.funder_rors).to eq(["https://ror.org/00k4n6c32", "https://ror.org/00a0jsq62"])
+      expect(doi.as_indexed_json["funder_rors"]).to eq(["https://ror.org/00k4n6c32", "https://ror.org/00a0jsq62"])
+    end
+
+    it "has funder ancestor ROR in funder_parent_rors" do
+      expect(doi.funder_parent_rors).to eq(["https://ror.org/019w4f821", "https://ror.org/04cw6st05"])
+      expect(doi.as_indexed_json["funder_parent_rors"]).to eq(["https://ror.org/019w4f821", "https://ror.org/04cw6st05"])
+    end
+  end
 end

spec/concerns/rorable_spec.rb

@@ -653,6 +653,132 @@ def clear_doi_index
     end
   end
 
+  describe "GET /dois with funded_by filter", prefix_pool_size: 3 do
+    let!(:dois) { create_list(:doi, 10, client: client, aasm_state: "findable", version_info: "testtag") }
+    let!(:funded_by_ec_dois) do
+      create_list(:doi, 5, client: client, aasm_state: "findable", funding_references:
+        [
+          {
+            "awardUri": "info:eu-repo/grantAgreement/EC/FP7/282625/",
+            "awardTitle": "MOTivational strength of ecosystem services and alternative ways to express the value of BIOdiversity",
+            "funderName": "European Commission",
+            "awardNumber": "282625",
+            "funderIdentifier": "https://doi.org/10.13039/501100000780",
+            "funderIdentifierType": "Crossref Funder ID"
+          }
+      ])
+    end
+
+    let!(:funded_by_ec_crossref_funder_id_descendant_dois) do
+      create_list(:doi, 2, client: client, aasm_state: "findable", funding_references:
+        [
+          {
+            "awardUri": "info:eu-repo/grantAgreement/EC/FP7/282625/",
+            "awardTitle": "MOTivational strength of ecosystem services and alternative ways to express the value of BIOdiversity",
+            "funderName": "European Commission",
+            "awardNumber": "282625",
+            "funderIdentifier": "https://doi.org/10.13039/501100000781",
+            "funderIdentifierType": "Crossref Funder ID"
+          }
+      ])
+    end
+    let!(:funded_by_ec_ror_descendant_dois) do
+      create_list(:doi, 4, client: client, aasm_state: "findable", funding_references:
+        [
+          {
+            "awardUri": "info:eu-repo/grantAgreement/EC/FP7/282625/",
+            "awardTitle": "MOTivational strength of ecosystem services and alternative ways to express the value of BIOdiversity",
+            "funderName": "European Commission",
+            "awardNumber": "282625",
+            "funderIdentifier": "ror.org/05mkt9r11",
+            "funderIdentifierType": "ROR"
+          }
+      ])
+    end
+    let!(:funded_by_lshtm_dois) do
+      create_list(:doi, 2, client: client, aasm_state: "findable", funding_references:
+        [
+          {
+            "awardUri": "info:eu-repo/grantAgreement/EC/FP7/284382/",
+            "awardTitle": "Institutionalizing global genetic-resource commons. Global Strategies for accessing and using essential public knowledge assets in the life sciences.",
+            "funderName": "London School of Hygiene & Tropical Medicine",
+            "awardNumber": "284382",
+            "funderIdentifier": "https://ror.org/00a0jsq62",
+            "funderIdentifierType": "ROR"
+          }
+      ])
+    end
+
+    before do
+      clear_doi_index
+      import_doi_index
+    end
+
+    it "filters by funder when funded_by is set with Crossref Funder ID funderIdentifier", vcr: true do
+      get "/dois?funded-by=https://ror.org/00k4n6c32", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(5)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+    end
+
+    it "filters by funder when query string is set with Crossref Funder ID funderIdentifier", vcr: true do
+      get "/dois?query=funder_rors:\"https://ror.org/00k4n6c32\"", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(5)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+    end
+
+    it "filters by funder when funded_by is set with ROR funderIdentifier", vcr: true do
+      get "/dois?funded-by=https://ror.org/00a0jsq62", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(2)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_lshtm_dois.first.doi.downcase)
+    end
+
+    it "filters by funder and funder child orgs when funded_by is set with Crossref Funder ID funderIdentifier and include_funder_child_organizations is true", vcr: true do
+      get "/dois?funded-by=https://ror.org/00k4n6c32&include-funder-child-organizations=true", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(11)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_crossref_funder_id_descendant_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_ror_descendant_dois.first.doi.downcase)
+    end
+
+    it "filters by funder and funder child orgs when query string is set with Crossref Funder ID funderIdentifier and child organizations are included", vcr: true do
+      get "/dois?query=funder_rors:\"https://ror.org/00k4n6c32\" OR funder_parent_rors:\"https://ror.org/00k4n6c32\"", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(11)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_crossref_funder_id_descendant_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_ror_descendant_dois.first.doi.downcase)
+    end
+
+    it "filters by funder when entered ROR is a parent of entered RORs or Crossref Funder IDs but isn't included in metadata", vcr: true do
+      get "/dois?funded-by=https://ror.org/019w4f821&include-funder-child-organizations=true", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(11)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_crossref_funder_id_descendant_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_ror_descendant_dois.first.doi.downcase)
+    end
+
+    it "filters by funder when entered ROR in query is a parent of entered RORs or Crossref Funder IDs but isn't included in metadata", vcr: true do
+      get "/dois?query=funder_rors:\"https://ror.org/019w4f821\" OR funder_parent_rors:\"https://ror.org/019w4f821\"", nil, headers
+
+      expect(last_response.status).to eq(200)
+      expect(json["data"].size).to eq(11)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_crossref_funder_id_descendant_dois.first.doi.downcase)
+      expect(json["data"].map { |d| d["attributes"]["doi"] }).to include(funded_by_ec_ror_descendant_dois.first.doi.downcase)
+    end
+  end
+
   describe "GET /dois with resource-type-id filter", elasticsearch: false, prefix_pool_size: 1 do
     let!(:instrument_doi) { create(:doi, client: client, aasm_state: "findable", types: { "resourceTypeGeneral": "Instrument" }) }
     let!(:study_registration_doi) { create(:doi, client: client, aasm_state: "findable", types: { "resourceTypeGeneral": "StudyRegistration" }) }
@@ -726,6 +852,7 @@ def clear_doi_index
     let!(:dois) { create_list(:doi, 3, aasm_state: "findable") }
 
     before do
+      clear_doi_index
       import_doi_index
     end