diff --git a/_includes/repo_summary.html b/_includes/repo_summary.html index d4603af0..fe5781e6 100644 --- a/_includes/repo_summary.html +++ b/_includes/repo_summary.html @@ -4,6 +4,10 @@ {% endcomment %} + + + + diff --git a/_layouts/search_packages.html b/_layouts/search_packages.html index 132a8b9b..2f0dc586 100644 --- a/_layouts/search_packages.html +++ b/_layouts/search_packages.html @@ -246,6 +246,7 @@

More resources

const LEFT_ARROW = "\u2B05"; const RIGHT_ARROW = "\u27A1"; const CIRCLED_DOT = "\u2299"; + const BOLT = "\u26A1"; const isRepos = getIsRepos(); const tableSort = searchArray ? SCORE_SORT : (isRepos ? REPO_SORT : PACKAGE_SORT); @@ -273,7 +274,7 @@

More resources

{ title:"Package", field:"url", width:200, formatter:"link", formatterParams:{labelField:"package", urlPrefix:"{{site.baseurl}}"}}, { title:"Description", field:"description", minWidth:isRepos ? 160: 300}, - { title:STAR, field:"released", width:40, formatter:"tickCross", headerTooltip:"released: release status", hozAlign:"center", + { title:BOLT, field:"released", width:40, formatter:"tickCross", headerTooltip:"released: release status", hozAlign:"center", formatterParams: {allowTruthy: true, allowEmpty: true}}, { title:LEFT_ARROW, field:"pkg_deps", width:40, headerTooltip:"pkg_deps: package dependency count"}, { title:RIGHT_ARROW, field:"dependants", width:40, headerTooltip:"dependants: package used by count"}, @@ -281,6 +282,10 @@

More resources

hozAlign:"center", formatterParams: {allowTruthy: true}, sorter:function(a, b, aRow, bRow, column, dir, sorterParams) { return sorterParams.indexOf(a) - sorterParams.indexOf(b); }, sorterParams:['ros_core', 'ros_base', 'desktop', 'desktop_full']}, + { title:STAR, field:"stars", width:40, sorter:"number", headerTooltip:"Repo stars", formatter:function(cell, formatterParams, onRendered) { + const value = cell.getValue(); + return value >= 10000 ? Math.floor(value / 1000).toString() + "K" : value.toString(); + }}, { title:"Authors", field:"authors", width:120}, { title:"Maintainers", field:"maintainers", width:120}, { title:"Org", field:"org", width:100}, @@ -313,7 +318,8 @@

More resources

tableOptions['groupHeader'] = (value, count, data, group) => { header = `${data[0]['last_commit_time']}` + - `${value}`; + `${value}` + + `${data[0]['repo_description']}` return header; } } diff --git a/_plugins/rosindex_generator.rb b/_plugins/rosindex_generator.rb index 934ed71c..c6419358 100644 --- a/_plugins/rosindex_generator.rb +++ b/_plugins/rosindex_generator.rb @@ -7,6 +7,7 @@ require 'colorator' require 'fileutils' require 'find' +require 'nokogiri' require 'rexml/document' require 'rexml/xpath' require 'pathname' @@ -30,6 +31,7 @@ $fetched_uris = {} $debug = false +$repo_scrape = {} DEFAULT_LANGUAGE_PREFIX = 'en' HEAVY_CHECKMARK = "\u2714" HEAVY_MINUS = "\u2796" @@ -647,6 +649,69 @@ def find_packages(site, distro, repo, snapshot, local_path) return packages end + def fetch(uri_str, limit = 10) + # get an http response, accounting for redirects. + if limit <= 0 then + raise StandardError.new "Redirect limit exceeded for #{uri_str}" + end + response = Net::HTTP.get_response(URI(uri_str)) + case response + when Net::HTTPSuccess + response + when Net::HTTPRedirection + fetch(response['location'], limit - 1) + else + response.error! + end + end + + def scrape_repo_page(uri_s) + # Scrapes a github repository home page to get various items + begin + # cache the results since this only depends on the repo uri + if $repo_scrape.key?(uri_s) then + return $repo_scrape[uri_s] + end + + repo_uri = URI(uri_s) + if repo_uri.host == 'github.com' then + response = fetch(uri_s) + document = Nokogiri::HTML(response.body) + element = document.at(".Layout-sidebar .octicon-star + strong") + if element then + star_count_f = element.text.to_f + if element.text.include? 'k' then + star_count_f = 1000 * star_count_f + end + star_count = star_count_f.to_i + else + star_count = nil + end + element = document.at('.Layout-sidebar p') + description = if element then element.text.strip else '' end + tag_elements = document.css('h3:contains("Topics") + div a') + tags = [] + tag_elements.each do |element| + tags.push(element.text.strip) + end + repo_parms = { + stars: star_count, + description: description, + tags: tags, + } + else + repo_parms = {} + end + + rescue => e + puts "Error in scrape_repo_page: #{ e.message } for #{ uri_s }" + repo_parms = {} + end + + $repo_scrape[uri_s] = repo_parms + return repo_parms +end + # scrape a version of a repository for packages and their contents def scrape_version(site, repo, distro, snapshot, vcs) @@ -656,6 +721,7 @@ def scrape_version(site, repo, distro, snapshot, vcs) end # initialize this snapshot data + repo_page = scrape_repo_page(repo.uri) data = snapshot.data = { # get the uri for resolving raw links (for imgages, etc) 'raw_uri' => get_raw_uri(repo.uri, repo.type, snapshot.version), @@ -665,7 +731,11 @@ def scrape_version(site, repo, distro, snapshot, vcs) 'readme' => nil, 'readme_rendered' => nil, 'contributing' => nil, - 'contributing_rendered' => nil} + 'contributing_rendered' => nil, + 'stars' => repo_page.fetch(:stars, ''), + 'description' => repo_page.fetch(:description, ''), + 'tags' => repo_page.fetch(:tags, []), + } # load the repo readme for this branch if it exists data['readme_rendered'], data['readme'] = get_readme( @@ -698,7 +768,10 @@ def scrape_version(site, repo, distro, snapshot, vcs) snapshot.packages[package_name] = package # collect tags from discovered packages - repo.tags = Set.new(repo.tags).merge(package_data['tags']).to_a + repo.tags = Set.new(repo.tags).merge(package_data['tags']) + + # add any tags placed on a repo + repo.tags = repo.tags.merge(data['tags']).to_a # collect wiki data package.data['wiki'] = @wiki_data[package_name] @@ -1580,7 +1653,9 @@ def generate(site) 'pkg_deps' => p['pkg_deps'].length, 'dependants' => p['dependants'].length, 'readme' => readme_filtered, - 'org' => URI(repo.uri).path.split('/')[1] + 'org' => URI(repo.uri).path.split('/')[1], + 'stars' => repo_snapshot.data['stars'], + 'repo_description': repo_snapshot.data['description'], } dputs 'indexed: ' << "#{package_name} #{instance_id} #{distro}" diff --git a/help/package_list.md b/help/package_list.md index 20d492a6..c3b05e81 100644 --- a/help/package_list.md +++ b/help/package_list.md @@ -13,11 +13,12 @@ breadcrumbs: ['help'] - Package: Package name. - Description: Package description. -- Release status (★): Release status of the package. A checkmark signifies the package is released. +- Release status (⚡): Release status of the package. A checkmark signifies the package is released. - Core (⊙): Is this a dependency of a core package? (a dependent of ros_core, ros_base, desktop, or desktop_full). - Last commit date (📅): The last date a commit was made to the package's repository. - Package dependency count (⬅): How many packages this package depends on. - Package used by count(➡): How many packages use this package as a dependency. +- Stars (★): Github count of repo stars. - Authors: Names listed in package.xml as authors. - Maintainers: Names listed in package.xml as maintainers. - Repo: Repository containing this package.
Description{{snapshot.data.description}}
Checkout URI {{repo.uri}}