Merge pull request #260 from sensu/issue_256

pzupan · web-flow · commit 20c603b4170e · 2020-01-07T15:03:42.000-08:00
allow for readme override
diff --git a/app/helpers/extension_versions_helper.rb b/app/helpers/extension_versions_helper.rb
@@ -64,8 +64,9 @@ def extension_atom_content(extension_version)
   #
   def render_document(content, extension, repo_loc = "", version = "", hard_wrap: false)
     document = begin
-      if %w(md mdown markdown).include?(extension.downcase)
+      if ExtensionVersion::MARKDOWN_EXTENSIONS.include?(extension.downcase)
         filter = HTML::Pipeline.new [
+          HTML::Pipeline::PlainTextInputFilter,
           HTML::Pipeline::MarkdownFilter,
           HTML::Pipeline::AutolinkFilter,
           HTML::Pipeline::MentionFilter,
diff --git a/app/models/extension_version.rb b/app/models/extension_version.rb
@@ -1,5 +1,7 @@
 class ExtensionVersion < ApplicationRecord
   include SeriousErrors
+
+  MARKDOWN_EXTENSIONS = %w(markdown mdown mkdn mkd md)
   
   # Associations
   # --------------------
diff --git a/app/workers/sync_extension_contents_at_versions_worker.rb b/app/workers/sync_extension_contents_at_versions_worker.rb
@@ -1,4 +1,5 @@
 class SyncExtensionContentsAtVersionsWorker < ApplicationWorker
+  include MarkdownHelper
 
   def logger
     if Rails.env.production?
@@ -13,7 +14,7 @@ def perform(extension_id, tags, compatible_platforms = [], release_infos_by_tag
     @extension = Extension.find_by(id: extension_id)
     raise RuntimeError.new("#{I18n.t('nouns.extension')} ID: #{extension_id.inspect} not found.") unless @extension
 
-    puts "PERFORMING #{@extension.name}: #{tags.join(', ')}"
+    #puts "PERFORMING #{@extension.name}: #{tags.join(', ')}"
     logger.info("PERFORMING: #{@extension.inspect}, #{tags.inspect}, #{compatible_platforms.inspect}")
 
     @errored_tags = []
@@ -49,6 +50,7 @@ def sync_extension_version(release_info)
     set_commit_count(version)
     scan_files(version)
     sync_release_info(version, release_info)
+    check_for_overrides(version)
     PersistAssets.call(version: version)
     update_annotations(version)
     version.save
@@ -65,14 +67,14 @@ def semver?
 
     begin
       tag = SemverNormalizer.call(@tag)
-      puts "Normalized Tag: #{tag}"
+      # puts "Normalized Tag: #{tag}"
       Semverse::Version.new(tag)
       return true
     rescue Semverse::InvalidVersionFormat => error
       unless @errored_tags.include?(@tag)
         @errored_tags << @tag
         compilation_error = [@extension.compilation_error, error.message]
-        @extension.update_column(:compilation_error, compilation_error.reject(&:empty?).join('; '))
+        @extension.update_column(:compilation_error, compilation_error.compact.join('; '))
         message = "#{@extension.lowercase_name} release is invalid: #{error.message}"
         logger.info message
       end
@@ -100,6 +102,72 @@ def fetch_readme
     end
   end
 
+  def check_for_overrides(version)
+    return if version.blank? || version.config["overrides"].nil?
+    overrides = version.config["overrides"][0]
+    if overrides["readme_url"].present?
+      override_readme(version, overrides["readme_url"])
+    end
+  end
+
+  def override_readme(version, readme_url)
+
+    message = []
+
+    begin
+      url = URI.parse(readme_url)
+    rescue URI::Error => error
+      logger.info "URI error: #{readme_url} - #{error.message}"
+      return
+    end
+
+    readme_ext = File.extname(url.path).gsub(".", "")
+    unless ExtensionVersion::MARKDOWN_EXTENSIONS.include?(readme_ext)
+      message << "#{version.version} override readme_url is not a valid markdown file."
+    end
+
+    # get file contents
+    begin
+      file = url.open
+    rescue OpenURI::HTTPError => error
+      status = error.io.status
+      message << "#{version.version} #{url.path} file read error: #{status[0]} - #{status[1]}"
+      logger.info message.compact.join('; ')
+      compilation_error = [version.compilation_error] + message
+      version.update_column(:compilation_error, compilation_error.compact.join('; '))
+      return
+    end
+    
+    readme = file.read
+
+    if readme.include?('!DOCTYPE html')
+      message << "#{version.version} override readme is not valid markdown."
+    end
+
+    begin
+      filter = HTML::Pipeline.new [
+          HTML::Pipeline::MarkdownFilter,
+        ], {gfm: true}
+      filter.call(readme)
+    rescue
+      message << "#{version.version} override readme is not valid markdown."
+    end
+
+    if message.present?
+      compilation_error = [version.compilation_error] + message
+      version.update_column(:compilation_error, compilation_error.compact.join('; '))
+    else
+
+      readme = readme.encode(Encoding.find('UTF-8'), {invalid: :replace, undef: :replace, replace: ''})
+      
+      version.update(
+        readme: readme,
+        readme_extension: readme_ext
+      )
+      logger.info "OVERRIDE README: #{url.path}"
+    end
+  end
+
   def extract_readme_file_extension(filename)
     if match = filename.match(/\.[a-zA-Z0-9]+$/)
       match[0].gsub(".", "")
diff --git a/spec/interactors/compile_github_extension_version_config_spec.rb b/spec/interactors/compile_github_extension_version_config_spec.rb
@@ -13,7 +13,7 @@
                             "asset_filename"=>"\#{repo}-\#{version}-linux-x86_64.tar.gz"}],
                         "labels"=> {"example"=> "custom"},
                         "annotations"=> {"io.sensu.bonsai.test"=> "test value"},
-                        "description"=> "test asset"
+                        "description"=>"test asset"
                      } }
   let(:extension)    { create :extension, extension_versions_count: 0, github_url: "https://github.com/owner/#{repo_name}" }
   let(:version)      { create :extension_version, extension: extension, config: config, version: version_name }
diff --git a/spec/interactors/find_or_create_release_asset_spec.rb b/spec/interactors/find_or_create_release_asset_spec.rb
@@ -2,8 +2,6 @@
 
 describe FindOrCreateReleaseAsset do
 
-	
-
 	describe ".call" do
 
 		context "when valid version and build" do
diff --git a/spec/workers/sync_extension_contents_at_versions_worker_spec.rb b/spec/workers/sync_extension_contents_at_versions_worker_spec.rb
@@ -28,11 +28,11 @@
     @s3_bucket = s3.bucket(ENV['AWS_S3_ASSETS_BUCKET'])
 
     # trap incorrect ENV variables (Travis) or no network connection
-    begin
-      stub_aws unless @s3_bucket.exists?
-    rescue
+    #begin
+    #  stub_aws unless @s3_bucket.exists?
+    #rescue
       stub_aws
-    end
+    #end
     
   end
 
@@ -41,6 +41,8 @@ def stub_aws
     Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :exists?).and_return(true)
     Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :public_url).and_return('https://s3.us-west-2.amazonaws.com/bucket/example.com')
     Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :last_modified).and_return(DateTime.now.to_s(:db))
+    Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :delete).and_return(true)
+    Aws::S3::Resource.any_instance.stub_chain(:bucket, :object, :put).and_return(true)
   end
 
   describe 'tag checking' do
@@ -92,7 +94,7 @@ def stub_aws
     it 'puts annotations in annotations field' do
       config_hash['annotations'] = {
         suggested_asset_url: '/suggested/asset', 
-        suggested_asset_message: 'Suggested Asset Messaage'
+        suggested_asset_message: 'Suggested Asset Message'
       }
       version.update_column(:config, config_hash)
       subject.perform(extension.id, tags)
@@ -103,4 +105,21 @@ def stub_aws
 
   end
 
+  describe 'overrides' do 
+    let(:config_hash) { version.config }
+    let(:tags) { [version.version] }
+      
+    it 'loads an alternate readme file' do
+      config_hash["overrides"] = {
+        "readme_url"=>"https://github.com/sensu/bonsai/blob/master/README.md"
+      }
+      version.update_column(:config, config_hash)
+      subject.perform(extension.id, tags)
+      version.reload 
+      expect(version.readme).to include('bonsai.sensu.io')
+    end
+
+  end
+
+
 end
