diff --git a/Gemfile b/Gemfile index 31b57064..5b8fae70 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ group :docs do gem 'unix_utils', require: false gem 'tty-pager', require: false gem 'net-sftp', '>= 2.1.3.rc2', require: false + gem 'terminal-table', require: false end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index d968a27b..3ed8570b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,6 +101,8 @@ GEM unicode-display_width (~> 1.4.0) unicode_utils (~> 1.4.0) strings-ansi (0.1.0) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) @@ -153,6 +155,7 @@ DEPENDENCIES sinatra-contrib sprockets sprockets-helpers + terminal-table thin thor tty-pager diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index a7e388a8..b124c6db 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -132,7 +132,7 @@ module Docs end end - def get_latest_version(&block) + def get_latest_version(options, &block) raise NotImplementedError end @@ -147,15 +147,15 @@ module Docs # 1 -> 2 = outdated # 1.1 -> 1.2 = outdated # 1.1.1 -> 1.1.2 = not outdated - def is_outdated(current_version, latest_version) - current_parts = current_version.split(/\./).map(&:to_i) + def is_outdated(scraper_version, latest_version) + scraper_parts = scraper_version.split(/\./).map(&:to_i) latest_parts = latest_version.split(/\./).map(&:to_i) # Only check the first two parts, the third part is for patch updates [0, 1].each do |i| - break if i >= current_parts.length or i >= latest_parts.length - return true if latest_parts[i] > current_parts[i] - return false if latest_parts[i] < current_parts[i] + break if i >= scraper_parts.length or i >= latest_parts.length + return true if latest_parts[i] > scraper_parts[i] + return false if latest_parts[i] < scraper_parts[i] end false @@ -231,38 +231,62 @@ module Docs {} end + # # Utility methods for get_latest_version + # - def fetch(url, &block) - Request.run(url) do |response| + def fetch(url, options, &block) + headers = {} + + if options.key?(:github_token) and url.start_with?('https://api.github.com/') + headers['Authorization'] = "token #{options[:github_token]}" + end + + options[:logger].debug("Fetching #{url}") + + Request.run(url, { headers: headers }) do |response| if response.success? block.call response.body else + options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") block.call nil end end end - def fetch_doc(url, &block) - fetch(url) do |body| - parser = Parser.new(body) - block.call parser.html + def fetch_doc(url, options, &block) + fetch(url, options) do |body| + block.call Nokogiri::HTML.parse body, nil, 'UTF-8' end end - def fetch_json(url, &block) - fetch(url) do |body| + def fetch_json(url, options, &block) + fetch(url, options) do |body| json = JSON.parse(body) block.call json end end - def get_npm_version(package, &block) - fetch_json("https://registry.npmjs.com/#{package}") do |json| + def get_npm_version(package, options, &block) + fetch_json("https://registry.npmjs.com/#{package}", options) do |json| block.call json['dist-tags']['latest'] end end + def get_latest_github_release(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) + end + + def get_github_tags(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) + end + + def get_github_file_contents(owner, repo, path, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| + block.call(Base64.decode64(json['content'])) + end + end + module FixInternalUrlsBehavior def self.included(base) base.extend ClassMethods diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index fa03eb36..059b0e8e 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -155,8 +155,8 @@ module Docs end end - def get_latest_version(&block) - get_npm_version('@angular/core', &block) + def get_latest_version(options, &block) + get_npm_version('@angular/core', options, &block) end private diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb index aa74ca1c..b6e18325 100644 --- a/lib/docs/scrapers/angularjs.rb +++ b/lib/docs/scrapers/angularjs.rb @@ -70,8 +70,8 @@ module Docs self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end - def get_latest_version(&block) - get_npm_version('angular', &block) + def get_latest_version(options, &block) + get_npm_version('angular', options, &block) end end end diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb index 60fb1953..293f74a7 100644 --- a/lib/docs/scrapers/ansible.rb +++ b/lib/docs/scrapers/ansible.rb @@ -88,8 +88,8 @@ module Docs list_of_all_modules.html) end - def get_latest_version(&block) - fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc| block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip end end diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb index 5eca041e..ba0fa340 100644 --- a/lib/docs/scrapers/apache.rb +++ b/lib/docs/scrapers/apache.rb @@ -34,8 +34,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(&block) - fetch_doc('http://httpd.apache.org/docs/') do |doc| + def get_latest_version(options, &block) + fetch_doc('http://httpd.apache.org/docs/', options) do |doc| block.call doc.at_css('#apcontents > ul a')['href'][0...-1] end end diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb index 15c477bf..5454140b 100644 --- a/lib/docs/scrapers/apache_pig.rb +++ b/lib/docs/scrapers/apache_pig.rb @@ -43,8 +43,8 @@ module Docs self.base_url = "https://pig.apache.org/docs/r#{release}/" end - def get_latest_version(&block) - fetch_doc('https://pig.apache.org/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://pig.apache.org/', options) do |doc| item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') block.call item.content.strip.sub(/Release /, '') end diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 930820b4..18e9bbbf 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -18,8 +18,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - fetch_doc('https://caolan.github.io/async/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://caolan.github.io/async/', options) do |doc| version = doc.at_css('#version-dropdown > a').content.strip[1..-1] block.call version end diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb index cc8bec6d..675f86be 100644 --- a/lib/docs/scrapers/babel.rb +++ b/lib/docs/scrapers/babel.rb @@ -23,8 +23,8 @@ module Docs '
' end - def get_latest_version(&block) - fetch_doc('https://babeljs.io/docs/en/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://babeljs.io/docs/en/', options) do |doc| block.call doc.at_css('a[href="/versions"] > h3').content end end diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb index 2fb7662f..ad6220e5 100644 --- a/lib/docs/scrapers/backbone.rb +++ b/lib/docs/scrapers/backbone.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - fetch_doc('https://backbonejs.org/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://backbonejs.org/', options) do |doc| version = doc.at_css('.version').content block.call version[1...-1] end diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb index b62868a6..5556f5b9 100644 --- a/lib/docs/scrapers/bash.rb +++ b/lib/docs/scrapers/bash.rb @@ -18,8 +18,8 @@ module Docs Licensed under the GNU Free Documentation License. HTML - def get_latest_version(&block) - fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body| + def get_latest_version(options, &block) + fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body| version = body.scan(/, Version ([0-9.]+)/)[0][0] block.call version[0...-1] end diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb index 73888004..8a960b87 100644 --- a/lib/docs/scrapers/bluebird.rb +++ b/lib/docs/scrapers/bluebird.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - get_npm_version('bluebird', &block) + def get_latest_version(options, &block) + get_npm_version('bluebird', options, &block) end end end diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb index 7b2406b8..aa0b4cc3 100644 --- a/lib/docs/scrapers/bootstrap.rb +++ b/lib/docs/scrapers/bootstrap.rb @@ -34,5 +34,11 @@ module Docs options[:only] = %w(getting-started/ css/ components/ javascript/) end + + def get_latest_version(options, &block) + fetch_doc('https://getbootstrap.com/', options) do |doc| + block.call doc.at_css('#bd-versions').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/bottle.rb b/lib/docs/scrapers/bottle.rb index 25ad7f6e..6e4a19a8 100644 --- a/lib/docs/scrapers/bottle.rb +++ b/lib/docs/scrapers/bottle.rb @@ -27,5 +27,12 @@ module Docs self.release = '0.11.7' self.base_url = "https://bottlepy.org/docs/#{self.version}/" end + + def get_latest_version(options, &block) + fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc| + label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') + block.call label.content.sub(/Bottle /, '') + end + end end end diff --git a/lib/docs/scrapers/bower.rb b/lib/docs/scrapers/bower.rb index b032f1d3..1102ee75 100644 --- a/lib/docs/scrapers/bower.rb +++ b/lib/docs/scrapers/bower.rb @@ -19,5 +19,9 @@ module Docs © 2018 Bower contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('bower', options, &block) + end end end diff --git a/lib/docs/scrapers/cakephp.rb b/lib/docs/scrapers/cakephp.rb index 08dbead0..b123ab7a 100644 --- a/lib/docs/scrapers/cakephp.rb +++ b/lib/docs/scrapers/cakephp.rb @@ -71,6 +71,12 @@ module Docs self.base_url = 'https://api.cakephp.org/2.7/' end + def get_latest_version(options, &block) + fetch_doc('https://api.cakephp.org/3.7/', options) do |doc| + block.call doc.at_css('.version-picker .dropdown-toggle').content.strip + end + end + private def parse(response) diff --git a/lib/docs/scrapers/chai.rb b/lib/docs/scrapers/chai.rb index 9d8aa4d2..422bd5a9 100644 --- a/lib/docs/scrapers/chai.rb +++ b/lib/docs/scrapers/chai.rb @@ -23,5 +23,9 @@ module Docs © 2016 Chai.js Assertion Library
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('chai', options, &block) + end end end diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb index 2fd32a83..337d1202 100644 --- a/lib/docs/scrapers/chef.rb +++ b/lib/docs/scrapers/chef.rb @@ -47,5 +47,12 @@ module Docs options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//] end + + def get_latest_version(options, &block) + fetch_doc('https://docs-archive.chef.io/', options) do |doc| + cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)') + block.call cell.content.sub(/Chef Client /, '') + end + end end end diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb index c6bdcdea..465a4493 100644 --- a/lib/docs/scrapers/clojure.rb +++ b/lib/docs/scrapers/clojure.rb @@ -27,5 +27,11 @@ module Docs self.release = '1.7' self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/' end + + def get_latest_version(options, &block) + fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc| + block.call doc.at_css('#header-version').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb index c455e4fd..dde4721c 100644 --- a/lib/docs/scrapers/cmake.rb +++ b/lib/docs/scrapers/cmake.rb @@ -59,5 +59,12 @@ module Docs self.release = '3.5.2' self.base_url = 'https://cmake.org/cmake/help/v3.5/' end + + def get_latest_version(options, &block) + fetch_doc('https://cmake.org/documentation/', options) do |doc| + link = doc.at_css('.entry-content ul > li > strong > a > big') + block.call link.content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb index 919f146d..2e28de7f 100644 --- a/lib/docs/scrapers/codeception.rb +++ b/lib/docs/scrapers/codeception.rb @@ -18,5 +18,11 @@ module Docs © 2011 Michael Bodnarchuk and contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://codeception.com/changelog', options) do |doc| + block.call doc.at_css('#page > h4').content + end + end end end diff --git a/lib/docs/scrapers/codeceptjs.rb b/lib/docs/scrapers/codeceptjs.rb index 13189340..e3f4fda8 100644 --- a/lib/docs/scrapers/codeceptjs.rb +++ b/lib/docs/scrapers/codeceptjs.rb @@ -21,5 +21,9 @@ module Docs © 2015 DavertMik <davert@codegyre.com> (http://codegyre.com)
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('codeceptjs', options, &block) + end end end diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb index 573f9b8c..864cf700 100644 --- a/lib/docs/scrapers/codeigniter.rb +++ b/lib/docs/scrapers/codeigniter.rb @@ -38,5 +38,12 @@ module Docs version '3' do self.release = '3.1.8' end + + def get_latest_version(options, &block) + fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc| + header = doc.at_css('#change-log h2') + block.call header.content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb index 23e9557f..d848d208 100644 --- a/lib/docs/scrapers/coffeescript.rb +++ b/lib/docs/scrapers/coffeescript.rb @@ -30,5 +30,9 @@ module Docs options[:container] = '.container' end + + def get_latest_version(options, &block) + get_npm_version('coffeescript', options, &block) + end end end diff --git a/lib/docs/scrapers/cordova.rb b/lib/docs/scrapers/cordova.rb index f74c72ff..efe8fb03 100644 --- a/lib/docs/scrapers/cordova.rb +++ b/lib/docs/scrapers/cordova.rb @@ -42,5 +42,14 @@ module Docs self.release = '6.5.0' self.base_url = 'https://cordova.apache.org/docs/en/6.x/' end + + def get_latest_version(options, &block) + fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc| + label = doc.at_css('#versionDropdown').content.strip + version = label.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end end end diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb index 29061a1d..e70317f2 100644 --- a/lib/docs/scrapers/crystal.rb +++ b/lib/docs/scrapers/crystal.rb @@ -34,5 +34,11 @@ module Docs HTML end } + + def get_latest_version(options, &block) + fetch('https://crystal-lang.org/api', options) do |body| + block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/d.rb b/lib/docs/scrapers/d.rb index 6126380e..b0adaf31 100644 --- a/lib/docs/scrapers/d.rb +++ b/lib/docs/scrapers/d.rb @@ -26,5 +26,11 @@ module Docs def initial_urls %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html) end + + def get_latest_version(options, &block) + fetch_doc('https://dlang.org/changelog/', options) do |doc| + block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id'] + end + end end end diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb index 26b27ca5..cfbbafc9 100644 --- a/lib/docs/scrapers/d3.rb +++ b/lib/docs/scrapers/d3.rb @@ -58,5 +58,9 @@ module Docs options[:root_title] = 'D3.js' options[:only_patterns] = [/\.md\z/] end + + def get_latest_version(options, &block) + get_npm_version('d3', options, &block) + end end end diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb index c345c22f..42d20423 100644 --- a/lib/docs/scrapers/dart.rb +++ b/lib/docs/scrapers/dart.rb @@ -31,5 +31,12 @@ module Docs self.release = '1.24.3' self.base_url = "https://api.dartlang.org/stable/#{release}/" end + + def get_latest_version(options, &block) + fetch_doc('https://api.dartlang.org/', options) do |doc| + label = doc.at_css('footer > span').content.strip + block.call label.sub(/Dart /, '') + end + end end end diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb index 45273540..746c0f40 100644 --- a/lib/docs/scrapers/django.rb +++ b/lib/docs/scrapers/django.rb @@ -63,5 +63,11 @@ module Docs self.release = '1.8.18' self.base_url = 'https://docs.djangoproject.com/en/1.8/' end + + def get_latest_version(options, &block) + fetch_doc('https://docs.djangoproject.com/', options) do |doc| + block.call doc.at_css('#doc-versions > li.current > span > strong').content + end + end end end diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb index 92494f8a..dd849391 100644 --- a/lib/docs/scrapers/docker.rb +++ b/lib/docs/scrapers/docker.rb @@ -137,5 +137,12 @@ module Docs options[:container] = '#docs' options[:only_patterns] << /\Aswarm\// end + + def get_latest_version(options, &block) + fetch_doc('https://docs.docker.com/', options) do |doc| + label = doc.at_css('.nav-container button.dropdown-toggle').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb index 937ed21a..66dccb6f 100644 --- a/lib/docs/scrapers/dojo.rb +++ b/lib/docs/scrapers/dojo.rb @@ -36,6 +36,12 @@ module Docs urls.map { |url| "#{url}" }.join end + def get_latest_version(options, &block) + fetch_doc('https://dojotoolkit.org/api/', options) do |doc| + block.call doc.at_css('#versionSelector > option[selected]').content + end + end + private def get_url_list(json, set = Set.new) diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb index 5710eb36..92da4193 100644 --- a/lib/docs/scrapers/drupal.rb +++ b/lib/docs/scrapers/drupal.rb @@ -98,5 +98,14 @@ module Docs /\A[\w\-\.]+\.php\/7\.x\z/ ] end + + def get_latest_version(options, &block) + fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc| + version = doc.at_css('td.form > form > select > option[selected]').content + version = version.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end end end diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb index 3cb399f0..dd3cf00a 100644 --- a/lib/docs/scrapers/electron.rb +++ b/lib/docs/scrapers/electron.rb @@ -22,5 +22,11 @@ module Docs © 2013–2018 GitHub Inc.
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://electronjs.org/docs', options) do |doc| + block.call doc.at_css('.docs-version').content + end + end end end diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index 10d5aac1..d5b8dbe6 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -97,5 +97,11 @@ module Docs 'https://elixir-lang.org/getting-started/' ] end + + def get_latest_version(options, &block) + fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc| + block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/ember.rb b/lib/docs/scrapers/ember.rb index 3db20c94..24a8817e 100644 --- a/lib/docs/scrapers/ember.rb +++ b/lib/docs/scrapers/ember.rb @@ -56,5 +56,11 @@ module Docs https://emberjs.com/api/ember-data/2.14/classes/DS ) end + + def get_latest_version(options, &block) + fetch_doc('https://emberjs.com/api/ember/release', options) do |doc| + block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip + end + end end end diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb index d6aa2a0b..7dcb0fae 100644 --- a/lib/docs/scrapers/erlang.rb +++ b/lib/docs/scrapers/erlang.rb @@ -55,5 +55,11 @@ module Docs version '18' do self.release = '18.3' end + + def get_latest_version(options, &block) + fetch_doc('https://www.erlang.org/downloads', options) do |doc| + block.call doc.at_css('.col-lg-3 > ul > li').content.strip + end + end end end diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb index 8b4c9a2e..dac9c283 100644 --- a/lib/docs/scrapers/eslint.rb +++ b/lib/docs/scrapers/eslint.rb @@ -20,5 +20,9 @@ module Docs © JS Foundation and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('eslint', options, &block) + end end end diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb index 0fb4ed14..67ba07e8 100644 --- a/lib/docs/scrapers/express.rb +++ b/lib/docs/scrapers/express.rb @@ -28,5 +28,9 @@ module Docs © 2017 StrongLoop, IBM, and other expressjs.com contributors.
Licensed under the Creative Commons Attribution-ShareAlike License v3.0. HTML + + def get_latest_version(options, &block) + get_npm_version('express', options, &block) + end end end diff --git a/lib/docs/scrapers/falcon.rb b/lib/docs/scrapers/falcon.rb index 5bfd8efc..cd5b70cd 100644 --- a/lib/docs/scrapers/falcon.rb +++ b/lib/docs/scrapers/falcon.rb @@ -33,5 +33,11 @@ module Docs self.release = '1.2.0' self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/" end + + def get_latest_version(options, &block) + fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc| + block.call doc.at_css('#changelogs ul > li > a').content + end + end end end diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb index 5ccfa71c..9340961a 100644 --- a/lib/docs/scrapers/fish.rb +++ b/lib/docs/scrapers/fish.rb @@ -46,5 +46,11 @@ module Docs self.release = '2.2.0' self.base_url = "https://fishshell.com/docs/#{version}/" end + + def get_latest_version(options, &block) + fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc| + block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb index 16ea70dd..546473f7 100644 --- a/lib/docs/scrapers/flow.rb +++ b/lib/docs/scrapers/flow.rb @@ -18,5 +18,9 @@ module Docs © 2013–present Facebook Inc.
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('flow-bin', options, &block) + end end end diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb index 26b2da95..f10473d0 100644 --- a/lib/docs/scrapers/git.rb +++ b/lib/docs/scrapers/git.rb @@ -19,5 +19,11 @@ module Docs © 2005–2018 Linus Torvalds and others
Licensed under the GNU General Public License version 2. HTML + + def get_latest_version(options, &block) + fetch_doc('https://git-scm.com/', options) do |doc| + block.call doc.at_css('.version').content.strip + end + end end end diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb index be3bb54e..3252dd6d 100644 --- a/lib/docs/scrapers/gnu/gcc.rb +++ b/lib/docs/scrapers/gnu/gcc.rb @@ -99,5 +99,12 @@ module Docs options[:replace_paths] = CPP_PATHS end + + def get_latest_version(options, &block) + fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| + label = doc.at_css('ul > li > ul > li > a').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb index 2610178e..f72f7d65 100644 --- a/lib/docs/scrapers/gnu/gnu_fortran.rb +++ b/lib/docs/scrapers/gnu/gnu_fortran.rb @@ -25,5 +25,12 @@ module Docs self.release = '4.9.3' self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/" end + + def get_latest_version(options, &block) + fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| + label = doc.at_css('ul > li > ul > li > a').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb index 7b233317..6f8f7a4a 100644 --- a/lib/docs/scrapers/go.rb +++ b/lib/docs/scrapers/go.rb @@ -24,6 +24,15 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML + def get_latest_version(options, &block) + fetch_doc('https://golang.org/pkg/', options) do |doc| + footer = doc.at_css('#footer').content + version = footer.scan(/go([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end + private def parse(response) # Hook here because Nokogori removes whitespace from textareas diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb index 7e7da9a6..d43782c2 100644 --- a/lib/docs/scrapers/godot.rb +++ b/lib/docs/scrapers/godot.rb @@ -37,5 +37,11 @@ module Docs self.release = '2.1' self.base_url = "http://docs.godotengine.org/en/#{self.version}/" end + + def get_latest_version(options, &block) + fetch_doc('https://docs.godotengine.org/', options) do |doc| + block.call doc.at_css('.version').content.strip + end + end end end diff --git a/lib/docs/scrapers/graphite.rb b/lib/docs/scrapers/graphite.rb index 49ade898..d1d8b9d1 100644 --- a/lib/docs/scrapers/graphite.rb +++ b/lib/docs/scrapers/graphite.rb @@ -17,5 +17,11 @@ module Docs © 2011–2016 The Graphite Project
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(options, &block) + fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc| + block.call doc.at_css('#release-notes li > a').content + end + end end end diff --git a/lib/docs/scrapers/grunt.rb b/lib/docs/scrapers/grunt.rb index 2201c043..1e8af9fb 100644 --- a/lib/docs/scrapers/grunt.rb +++ b/lib/docs/scrapers/grunt.rb @@ -26,5 +26,9 @@ module Docs © GruntJS Team
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('grunt-cli', options, &block) + end end end diff --git a/lib/docs/scrapers/handlebars.rb b/lib/docs/scrapers/handlebars.rb index 22935d21..7df63102 100644 --- a/lib/docs/scrapers/handlebars.rb +++ b/lib/docs/scrapers/handlebars.rb @@ -19,5 +19,9 @@ module Docs © 2011–2017 by Yehuda Katz
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('handlebars', options, &block) + end end end diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index 442339b3..383e1990 100755 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -68,5 +68,12 @@ module Docs options[:only_patterns] = [/\Alibraries\//] end + + def get_latest_version(options, &block) + fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc| + label = doc.at_css('.related > ul > li:last-child').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/haxe.rb b/lib/docs/scrapers/haxe.rb index 33f20b93..5a685efc 100644 --- a/lib/docs/scrapers/haxe.rb +++ b/lib/docs/scrapers/haxe.rb @@ -66,5 +66,12 @@ module Docs version 'Python' do self.base_url = 'https://api.haxe.org/python/' end + + def get_latest_version(options, &block) + fetch_doc('https://api.haxe.org/', options) do |doc| + label = doc.at_css('.container.main-content h1 > small').content + block.call label.sub(/version /, '') + end + end end end diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index fba79ec0..fef1ed05 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -19,5 +19,11 @@ module Docs © 2009–present Homebrew contributors
Licensed under the BSD 2-Clause License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('Homebrew', 'brew', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb index fa7fb81b..342ce107 100644 --- a/lib/docs/scrapers/immutable.rb +++ b/lib/docs/scrapers/immutable.rb @@ -54,5 +54,9 @@ module Docs JS capybara.html end + + def get_latest_version(options, &block) + get_npm_version('immutable', options, &block) + end end end diff --git a/lib/docs/scrapers/influxdata.rb b/lib/docs/scrapers/influxdata.rb index 6c83b66b..4fc98c16 100644 --- a/lib/docs/scrapers/influxdata.rb +++ b/lib/docs/scrapers/influxdata.rb @@ -46,5 +46,12 @@ module Docs © 2015 InfluxData, Inc.
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc| + label = doc.at_css('.navbar--current-product').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb index 82f3c9cf..5f38e3d5 100644 --- a/lib/docs/scrapers/jasmine.rb +++ b/lib/docs/scrapers/jasmine.rb @@ -17,5 +17,11 @@ module Docs © 2008–2017 Pivotal Labs
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('jasmine', 'jasmine', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/jekyll.rb b/lib/docs/scrapers/jekyll.rb index 1faaa9de..a6af352f 100644 --- a/lib/docs/scrapers/jekyll.rb +++ b/lib/docs/scrapers/jekyll.rb @@ -28,5 +28,11 @@ module Docs © 2008–2018 Tom Preston-Werner and Jekyll contributors
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://jekyllrb.com/docs/', options) do |doc| + block.call doc.at_css('.meta a').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb index f4ce944f..71efcf54 100644 --- a/lib/docs/scrapers/jest.rb +++ b/lib/docs/scrapers/jest.rb @@ -17,5 +17,11 @@ module Docs © 2014–present Facebook Inc.
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc| + block.call doc.at_css('header > a > h3').content + end + end end end diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb index 20aca0dc..dad609e7 100644 --- a/lib/docs/scrapers/jquery/jquery_core.rb +++ b/lib/docs/scrapers/jquery/jquery_core.rb @@ -22,5 +22,9 @@ module Docs /Selectors\/odd/i, /index/i ] + + def get_latest_version(options, &block) + get_npm_version('jquery', options, &block) + end end end diff --git a/lib/docs/scrapers/jquery/jquery_mobile.rb b/lib/docs/scrapers/jquery/jquery_mobile.rb index 8e5abf1c..53b2c624 100644 --- a/lib/docs/scrapers/jquery/jquery_mobile.rb +++ b/lib/docs/scrapers/jquery/jquery_mobile.rb @@ -16,5 +16,12 @@ module Docs options[:fix_urls] = ->(url) do url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/' end + + def get_latest_version(options, &block) + fetch_doc('https://jquerymobile.com/', options) do |doc| + label = doc.at_css('.download-box > .download-option:last-child > span').content + block.call label.sub(/Version /, '') + end + end end end diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb index 0c90fc1a..05c276e1 100644 --- a/lib/docs/scrapers/jquery/jquery_ui.rb +++ b/lib/docs/scrapers/jquery/jquery_ui.rb @@ -15,5 +15,9 @@ module Docs options[:fix_urls] = ->(url) do url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/' end + + def get_latest_version(options, &block) + get_npm_version('jquery-ui', options, &block) + end end end diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb index bb3781ca..39feca71 100644 --- a/lib/docs/scrapers/jsdoc.rb +++ b/lib/docs/scrapers/jsdoc.rb @@ -21,5 +21,11 @@ module Docs © 2011–2017 the contributors to the JSDoc 3 documentation project
Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML + + def get_latest_version(options, &block) + get_latest_github_release('jsdoc3', 'jsdoc', options) do |release| + block.call release['tag_name'] + end + end end end diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb index 5bc16b77..0875835a 100644 --- a/lib/docs/scrapers/julia.rb +++ b/lib/docs/scrapers/julia.rb @@ -49,5 +49,11 @@ module Docs html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html' end + + def get_latest_version(options, &block) + get_latest_github_release('JuliaLang', 'julia', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb index 663f7847..60af1540 100644 --- a/lib/docs/scrapers/knockout.rb +++ b/lib/docs/scrapers/knockout.rb @@ -33,5 +33,11 @@ module Docs © Steven Sanderson, the Knockout.js team, and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('knockout', 'knockout', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/koa.rb b/lib/docs/scrapers/koa.rb index 3ce79cac..4d90e30f 100644 --- a/lib/docs/scrapers/koa.rb +++ b/lib/docs/scrapers/koa.rb @@ -34,5 +34,9 @@ module Docs © 2018 Koa contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('koa', options, &block) + end end end diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb index 415393d1..7539212d 100644 --- a/lib/docs/scrapers/kotlin.rb +++ b/lib/docs/scrapers/kotlin.rb @@ -28,5 +28,11 @@ module Docs © 2010–2018 JetBrains s.r.o.
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(options, &block) + get_latest_github_release('JetBrains', 'kotlin', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index 5c88ae0f..cdf32732 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -133,5 +133,11 @@ module Docs url end end + + def get_latest_version(options, &block) + get_latest_github_release('laravel', 'laravel', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb index c8e2071c..24bc6142 100644 --- a/lib/docs/scrapers/leaflet.rb +++ b/lib/docs/scrapers/leaflet.rb @@ -39,5 +39,11 @@ module Docs self.base_url = "https://leafletjs.com/reference-#{release}.html" end + def get_latest_version(options, &block) + fetch_doc('https://leafletjs.com/index.html', options) do |doc| + link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first + block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/less.rb b/lib/docs/scrapers/less.rb index a0947e1a..00c884eb 100644 --- a/lib/docs/scrapers/less.rb +++ b/lib/docs/scrapers/less.rb @@ -21,5 +21,12 @@ module Docs © 2009–2016 The Core Less Team
Licensed under the Creative Commons Attribution License 3.0. HTML + + def get_latest_version(options, &block) + fetch_doc('http://lesscss.org/features/', options) do |doc| + label = doc.at_css('.footer-links > li').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/liquid.rb b/lib/docs/scrapers/liquid.rb index 9ebc4041..4630b2d1 100644 --- a/lib/docs/scrapers/liquid.rb +++ b/lib/docs/scrapers/liquid.rb @@ -19,5 +19,11 @@ module Docs © 2005, 2006 Tobias Luetke
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_github_tags('Shopify', 'liquid', options) do |tags| + block.call tags[0]['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb index 0461f7b7..5488b9ab 100644 --- a/lib/docs/scrapers/lodash.rb +++ b/lib/docs/scrapers/lodash.rb @@ -32,5 +32,11 @@ module Docs self.release = '2.4.2' self.base_url = "https://lodash.com/docs/#{release}" end + + def get_latest_version(options, &block) + fetch_doc('https://lodash.com/docs/', options) do |doc| + block.call doc.at_css('#version > option[selected]').content + end + end end end diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb index 7f23bded..019edbab 100644 --- a/lib/docs/scrapers/love.rb +++ b/lib/docs/scrapers/love.rb @@ -39,5 +39,11 @@ module Docs © 2006–2016 LÖVE Development Team
Licensed under the GNU Free Documentation License, Version 1.3. HTML + + def get_latest_version(options, &block) + fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc| + block.call doc.at_css('#mw-content-text table a').content + end + end end end diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb index 40a5c007..30af5523 100644 --- a/lib/docs/scrapers/lua.rb +++ b/lib/docs/scrapers/lua.rb @@ -26,5 +26,11 @@ module Docs self.release = '5.1.5' self.base_url = 'https://www.lua.org/manual/5.1/' end + + def get_latest_version(options, &block) + fetch_doc('https://www.lua.org/manual/', options) do |doc| + block.call doc.at_css('p.menubar > a').content + end + end end end diff --git a/lib/docs/scrapers/marionette.rb b/lib/docs/scrapers/marionette.rb index fea6617f..12de6d0c 100644 --- a/lib/docs/scrapers/marionette.rb +++ b/lib/docs/scrapers/marionette.rb @@ -38,5 +38,9 @@ module Docs html_filters.push 'marionette/entries_v2' end + + def get_latest_version(options, &block) + get_npm_version('backbone.marionette', options, &block) + end end end diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb index ddd1f9de..948955a6 100644 --- a/lib/docs/scrapers/matplotlib.rb +++ b/lib/docs/scrapers/matplotlib.rb @@ -64,5 +64,11 @@ module Docs "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/" ] end + + def get_latest_version(options, &block) + get_latest_github_release('matplotlib', 'matplotlib', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/meteor.rb b/lib/docs/scrapers/meteor.rb index b38d5dc2..02b81bc3 100644 --- a/lib/docs/scrapers/meteor.rb +++ b/lib/docs/scrapers/meteor.rb @@ -45,5 +45,11 @@ module Docs self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"] options[:fix_urls] = nil end + + def get_latest_version(options, &block) + fetch_doc('https://docs.meteor.com/#/full/', options) do |doc| + block.call doc.at_css('select.version-select > option').content + end + end end end diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb index 8ab9bdc8..6654d754 100644 --- a/lib/docs/scrapers/mocha.rb +++ b/lib/docs/scrapers/mocha.rb @@ -18,5 +18,9 @@ module Docs © 2011–2018 JS Foundation and contributors
Licensed under the Creative Commons Attribution 4.0 International License. HTML + + def get_latest_version(options, &block) + get_npm_version('mocha', options, &block) + end end end diff --git a/lib/docs/scrapers/modernizr.rb b/lib/docs/scrapers/modernizr.rb index 96c82153..93a738bb 100644 --- a/lib/docs/scrapers/modernizr.rb +++ b/lib/docs/scrapers/modernizr.rb @@ -15,5 +15,9 @@ module Docs © 2009–2017 The Modernizr team
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('modernizr', options, &block) + end end end diff --git a/lib/docs/scrapers/moment.rb b/lib/docs/scrapers/moment.rb index 88df0d14..9dd27107 100644 --- a/lib/docs/scrapers/moment.rb +++ b/lib/docs/scrapers/moment.rb @@ -22,5 +22,11 @@ module Docs © JS Foundation and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('http://momentjs.com/', options) do |doc| + block.call doc.at_css('.hero-title > h1 > span').content + end + end end end diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb index 71ee04d2..2d221990 100644 --- a/lib/docs/scrapers/mongoose.rb +++ b/lib/docs/scrapers/mongoose.rb @@ -26,5 +26,12 @@ module Docs © 2010 LearnBoost
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://mongoosejs.com/docs/', options) do |doc| + label = doc.at_css('.pure-menu-link').content.strip + block.call label.sub(/Version /, '') + end + end end end diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb index 761da1de..f676010d 100644 --- a/lib/docs/scrapers/rdoc/minitest.rb +++ b/lib/docs/scrapers/rdoc/minitest.rb @@ -21,5 +21,11 @@ module Docs © Ryan Davis, seattle.rb
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents| + block.call contents.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb index 6bdce34a..9cb2ab9b 100644 --- a/lib/docs/scrapers/rdoc/rails.rb +++ b/lib/docs/scrapers/rdoc/rails.rb @@ -93,5 +93,11 @@ module Docs version '4.1' do self.release = '4.1.16' end + + def get_latest_version(options, &block) + get_latest_github_release('rails', 'rails', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb index dd296765..292540db 100644 --- a/lib/docs/scrapers/rdoc/ruby.rb +++ b/lib/docs/scrapers/rdoc/ruby.rb @@ -84,5 +84,17 @@ module Docs version '2.2' do self.release = '2.2.10' end + + def get_latest_version(options, &block) + get_github_tags('ruby', 'ruby', options) do |tags| + tags.each do |tag| + version = tag['name'].gsub(/_/, '.')[1..-1] + if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 + block.call version + break + end + end + end + end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index eb3467f2..f370544c 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -1,4 +1,12 @@ class UpdatesCLI < Thor + # The GitHub user that is allowed to upload reports + # TODO: Update this before creating a PR + UPLOAD_USER = 'jmerle' + + # The repository to create an issue in when uploading the results + # TODO: Update this before creating a PR + UPLOAD_REPO = 'jmerle/devdocs' + def self.to_s 'Updates' end @@ -6,10 +14,14 @@ class UpdatesCLI < Thor def initialize(*args) require 'docs' require 'progress_bar' + require 'terminal-table' + require 'date' super end - desc 'check [--verbose] [doc]...', 'Check for outdated documentations' + desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations' + option :github_token, :type => :string + option :upload, :type => :boolean option :verbose, :type => :boolean def check(*names) # Convert names to a list of Scraper instances @@ -19,23 +31,26 @@ class UpdatesCLI < Thor # Check all documentations for updates when no arguments are given docs = Docs.all if docs.empty? - progress_bar = ::ProgressBar.new docs.length - progress_bar.write + opts = { + logger: logger + } - results = docs.map do |doc| - result = check_doc(doc) - progress_bar.increment! - result + if options.key?(:github_token) + opts[:github_token] = options[:github_token] end - valid_results = results.select {|result| result.is_a?(Hash)} + with_progress_bar do |bar| + bar.max = docs.length + bar.write + end - up_to_date_results = valid_results.select {|result| !result[:is_outdated]} - outdated_results = valid_results.select {|result| result[:is_outdated]} + results = docs.map do |doc| + result = check_doc(doc, opts) + with_progress_bar(&:increment!) + result + end - log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty? - logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty? - log_results('Outdated', outdated_results) unless outdated_results.empty? + process_results(results) rescue Docs::DocNotFound => error logger.error(error) logger.info('Run "thor docs:list" to see the list of docs.') @@ -43,53 +58,260 @@ class UpdatesCLI < Thor private - def check_doc(doc) + def check_doc(doc, options) # Newer scraper versions always come before older scraper versions - # Therefore, the first item's release value is the latest current scraper version + # Therefore, the first item's release value is the latest scraper version # # For example, a scraper could scrape 3 versions: 10, 11 and 12 # doc.versions.first would be the scraper for version 12 instance = doc.versions.first.new - return nil unless instance.class.method_defined?(:options) - - current_version = instance.options[:release] - return nil if current_version.nil? + scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil + return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil? logger.debug("Checking #{doc.name}") - instance.get_latest_version do |latest_version| + instance.get_latest_version(options) do |latest_version| return { name: doc.name, - current_version: current_version, + scraper_version: scraper_version, latest_version: latest_version, - is_outdated: instance.is_outdated(current_version, latest_version) + is_outdated: instance.is_outdated(scraper_version, latest_version) } end - - return nil rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") + error_result(doc, '`get_latest_version` is not implemented') rescue logger.error("Error while checking #{doc.name}") raise end - def log_results(label, results) - logger.info("#{label} documentations (#{results.length}):") + def error_result(doc, reason) + { + name: doc.name, + error: reason + } + end + + def process_results(results) + successful_results = results.select {|result| result.key?(:is_outdated)} + failed_results = results.select {|result| result.key?(:error)} + + up_to_date_results = successful_results.select {|result| !result[:is_outdated]} + outdated_results = successful_results.select {|result| result[:is_outdated]} + + log_results(outdated_results, up_to_date_results, failed_results) + upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload] + end + + # + # Result logging methods + # + + def log_results(outdated_results, up_to_date_results, failed_results) + log_failed_results(failed_results) unless failed_results.empty? + log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty? + log_successful_results('Outdated', outdated_results) unless outdated_results.empty? + end + + def log_successful_results(label, results) + title = "#{label} documentations (#{results.length})" + headings = ['Documentation', 'Scraper version', 'Latest version'] + rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]} + + table = Terminal::Table.new :title => title, :headings => headings, :rows => rows + puts table + end + + def log_failed_results(results) + title = "Documentations that could not be checked (#{results.length})" + headings = %w(Documentation Reason) + rows = results.map {|result| [result[:name], result[:error]]} + + table = Terminal::Table.new :title => title, :headings => headings, :rows => rows + puts table + end + + # + # Upload methods + # + + def upload_results(outdated_results, up_to_date_results, failed_results) + # We can't create issues without a GitHub token + unless options.key?(:github_token) + logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the --github-token parameter') + return + end + + logger.info('Uploading the results to a new GitHub issue') + + logger.info('Checking if the GitHub token belongs to the correct user') + github_get('/user') do |user| + # Only allow the DevDocs bot to upload reports + if user['login'] == UPLOAD_USER + issue = results_to_issue(outdated_results, up_to_date_results, failed_results) + + logger.info('Creating a new GitHub issue') + github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue| + search_params = { + q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", + sort: 'created', + order: 'desc' + } + + logger.info('Checking if the previous issue is still open') + github_get('/search/issues', search_params) do |matching_issues| + previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} + + if previous_issue.nil? + logger.info('No previous issue found') + log_upload_success(created_issue) + else + comment = "This report was superseded by ##{created_issue['number']}." + + logger.info('Commenting on the previous issue') + github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_| + if previous_issue['closed_at'].nil? + logger.info('Closing the previous issue') + github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_| + log_upload_success(created_issue) + end + else + logger.info('The previous issue has already been closed') + log_upload_success(created_issue) + end + end + end + end + end + else + logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + end + end + end + + def results_to_issue(outdated_results, up_to_date_results, failed_results) + results = [ + successful_results_to_markdown('Outdated', outdated_results), + successful_results_to_markdown('Up-to-date', up_to_date_results), + failed_results_to_markdown(failed_results) + ] + + results_str = results.select {|result| !result.nil?}.join("\n\n") + + title = "Documentation versions report for #{Date.today.strftime('%B')} 2019" + body = <<-MARKDOWN +## What is this? + +This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations. + +Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created. + +## Results + +The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are divided as follows: +- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated +- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch updates are ignored) +- #{failed_results.length} that could not be checked + MARKDOWN + + { + title: title, + body: body.strip + "\n\n" + results_str + } + end + + def successful_results_to_markdown(label, results) + return nil if results.empty? + + title = "#{label} documentations (#{results.length})" + headings = ['Documentation', 'Scraper version', 'Latest version'] + rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]} + + results_to_markdown(title, headings, rows) + end + + def failed_results_to_markdown(results) + return nil if results.empty? + + title = "Documentations that could not be checked (#{results.length})" + headings = %w(Documentation Reason) + rows = results.map {|result| [result[:name], result[:error]]} + + results_to_markdown(title, headings, rows) + end + + def results_to_markdown(title, headings, rows) + "
\n#{title}\n\n#{create_markdown_table(headings, rows)}\n
" + end + + def create_markdown_table(headings, rows) + header = headings.join(' | ') + separator = '-|' * headings.length + body = rows.map {|row| row.join(' | ')} + + header + "\n" + separator[0...-1] + "\n" + body.join("\n") + end + + def log_upload_success(created_issue) + logger.info("Successfully uploaded the results to #{created_issue['html_url']}") + end + + # + # HTTP utilities + # + + def github_get(endpoint, params = {}, &block) + github_request(endpoint, {method: :get, params: params}, &block) + end + + def github_post(endpoint, params, &block) + github_request(endpoint, {method: :post, body: params.to_json}, &block) + end + + def github_patch(endpoint, params, &block) + github_request(endpoint, {method: :patch, body: params.to_json}, &block) + end - results.each do |result| - logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}") + def github_request(endpoint, opts, &block) + url = "https://api.github.com#{endpoint}" + + # GitHub token authentication + opts[:headers] = { + Authorization: "token #{options[:github_token]}" + } + + # GitHub requires the Content-Type to be application/json when a body is passed + if opts.key?(:body) + opts[:headers]['Content-Type'] = 'application/json' end + + logger.debug("Making a #{opts[:method]} request to #{url}") + + Docs::Request.run(url, opts) do |response| + # response.success? is false if the response code is 201 + # GitHub returns 201 Created after an issue is created + if response.success? || response.code == 201 + block.call JSON.parse(response.body) + else + logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") + block.call nil + end + end + end + + # A utility method which ensures no progress bar is shown when stdout is not a tty + def with_progress_bar(&block) + return unless $stdout.tty? + @progress_bar ||= ::ProgressBar.new + block.call @progress_bar end def logger @logger ||= Logger.new($stdout).tap do |logger| logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO - logger.formatter = proc do |severity, datetime, progname, msg| - prefix = severity != "INFO" ? "[#{severity}] " : "" - "#{prefix}#{msg}\n" - end + logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"} end end end