Properly find CPAN provided modules

Author: NicholasBHubbard

lib/fpm/package/cpan.rb

@@ -53,7 +53,7 @@ def input(package)
       moduledir = package
       result = {}
     else
-      result = search(package)
+      result = search_module(package, version)
       tarball = download(result, version)
       moduledir = unpack(tarball)
     end
@@ -91,20 +91,17 @@ def input(package)
       else; metadata["license"]
     end
 
-    unless metadata["distribution"].nil?
+    unless metadata["release"].nil?
+      dist_name, _, dist_version = metadata["release"].rpartition('-')
       logger.info("Setting package name from 'distribution'",
-                   :distribution => metadata["distribution"])
-      self.name = fix_name(metadata["distribution"])
+                  :distribution => dist_name)
+      self.name = fix_name(dist_name)
+      self.provides = search_provided_modules(dist_name, dist_version)
     else
       logger.info("Setting package name from 'name'",
                    :name => metadata["name"])
       self.name = fix_name(metadata["name"])
-    end
-
-    unless metadata["module"].nil?
-      metadata["module"].each do |m|
-        self.provides << cap_name(m["name"]) + " = #{self.version}"
-      end
+      self.provides << cap_name(metadata["name"]) + " = #{self.version}"
     end
 
     # author is not always set or it may be a string instead of an array
@@ -167,7 +164,6 @@ def input(package)
             self.dependencies << "#{dep_name} >= #{version}"
             next
           end
-          dep = search(dep_name)
 
           name = cap_name(dep_name)
 
@@ -364,26 +360,80 @@ def download(metadata, cpan_version=nil)
     return build_path(tarball)
   end # def download
 
-  def search(package)
-    logger.info("Asking metacpan about a module", :module => package)
-    metacpan_url = "https://fastapi.metacpan.org/v1/module/" + package
+  def search_module(module_name, version=nil)
+    logger.info("Asking metacpan about a module", :module => module_name, :version => version)
+    metacpan_api_url = "https://fastapi.metacpan.org/v1/module/_search"
+    metacpan_api_query = <<-EOS
+{
+  "query": {
+    "bool": {
+      "must": [
+        { "term": { "module.name": "#{module_name}" } },
+        { "term": { "maturity": "released" } },
+        { "term": { #{version.nil? ? '"status": "latest"' : '"module.version": "' + version + '"'} } }
+      ]
+    }
+  },
+  "size": 1
+}
+EOS
     begin
-      response = httpfetch(metacpan_url)
+      response = httppost(metacpan_api_url, metacpan_api_query)
     rescue Net::HTTPServerException => e
       #logger.error("metacpan query failed.", :error => response.status_line,
                     #:module => package, :url => metacpan_url)
       logger.error("metacpan query failed.", :error => e.message,
-                    :module => package, :url => metacpan_url)
+                    :module => module_name, :version => version, :url => metacpan_api_url)
       raise FPM::InvalidPackageConfiguration, "metacpan query failed"
     end
 
     #data = ""
     #response.read_body { |c| p c; data << c }
     data = response.body
-    metadata = JSON.parse(data)
+    metadata = JSON.parse(data)['hits']['hits'][0]['_source']
     return metadata
   end # def search
 
+  def search_provided_modules(distribution, version)
+    logger.info("Asking metacpan about a releases provided modules",
+                :distribution => distribution,
+                :version => version)
+    metacpan_api_url = "https://fastapi.metacpan.org/v1/module/_search"
+    metacpan_api_query = <<-EOS
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "exists": { "field": "module" } },
+        { "term": { "release": "#{distribution}-#{version}" } }
+      ]
+    }
+  },
+  "fields": ["module.name", "module.version"],
+  "sort": [ { "module.name": { "order": "asc" } } ],
+  "size": 5000
+}
+EOS
+    begin
+      response = httppost(metacpan_api_url, metacpan_api_query)
+    rescue Net::HTTPServerException => e
+      logger.error("metacpan release query failed.", :error => e.message,
+                   :url => metacpan_api_url)
+      raise FPM::InvalidPackageConfiguration, "metacpan release query failed"
+    end
+    query_hits = JSON.parse(response.body)['hits']['hits']
+
+    provided_modules = []
+    query_hits.each do |m|
+      module_name = m["fields"]["module.name"]
+      module_version = m["fields"]["module.version"]
+      Array(module_name).zip(Array(module_version)).each do |name, version|
+        provided_modules << cap_name(name) + (version ? " = #{version}" : "")
+      end
+    end
+    return provided_modules
+  end # def search_provided_modules
+
   def cap_name(name)
     return "perl(" + name.gsub("-", "::") + ")"
   end # def cap_name

spec/fpm/package/cpan_spec.rb

@@ -43,17 +43,34 @@
   end
 
   it "should return metadata hash" do
-    metadata = subject.instance_eval { search("File::Temp") }
+    metadata = subject.instance_eval { search_module("File::Temp") }
     insist { metadata.class } == Hash
     insist { metadata["name"] } == "Temp.pm"
     insist { metadata["distribution"] } == "File-Temp"
   end
 
   it "should download precise version" do
-    metadata = subject.instance_eval { search("Set::Tiny") }
+    metadata = subject.instance_eval { search_module("Set::Tiny") }
     insist { File.basename(subject.instance_eval { download(metadata, "0.01") }) } == "Set-Tiny-0.01.tar.gz"
   end
 
+  it "should find distributions provided modules" do
+    provided_modules = subject.instance_eval { search_provided_modules("Test-DB", "0.10") }
+    insist { provided_modules } == ["perl(Test::DB) = 0.10", "perl(Test::DB::Mssql) = 0.10", "perl(Test::DB::Mysql) = 0.10", "perl(Test::DB::Postgres) = 0.10", "perl(Test::DB::Sqlite) = 0.10"]
+
+    # The Set-Tiny-0.01 release provides a single module, Set::Tiny version 0.01
+    provided_modules = subject.instance_eval { search_provided_modules("Set-Tiny", "0.01") }
+    insist { provided_modules } == ["perl(Set::Tiny) = 0.01"]
+
+    # Class::DI has packages with no version
+    provided_modules = subject.instance_eval { search_provided_modules("Class-DI", "0.03") }
+    insist { provided_modules } == ["perl(Class::DI) = 0.03", "perl(Class::DI::Definition)", "perl(Class::DI::Factory)", "perl(Class::DI::Resource)", "perl(Class::DI::Resource::YAML)"]
+
+    # File::Spec is a module provided by the PathTools distribution
+    provided_modules = subject.instance_eval { search_provided_modules("File-Spec", "3.75") }
+    insist { provided_modules } == []
+  end
+
   it "should package Digest::MD5" do
     # Set the version explicitly because we default to installing the newest
     # version, and a new version could be released that breaks the test.
@@ -71,6 +88,7 @@
     insist { subject.description } == "Perl interface to the MD-5 algorithm"
     insist { subject.vendor } == "Gisle Aas <gisle@activestate.com>"
     insist { subject.dependencies.sort } == ["perl >= 5.006", "perl(Digest::base) >= 1.00", "perl(XSLoader)"]
+    insist { subject.provides } == ["perl(Digest::MD5) = 2.58"]
   end
 
   it "should unpack tarball containing ./ leading paths" do