From d0300dd5851bcacbcba2c0ddacd057ada57b6cc2 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Fri, 8 Mar 2019 03:23:40 +0100 Subject: [PATCH 01/17] Create command to check for updates --- lib/docs/core/scraper.rb | 29 +++++++++++++++ lib/tasks/updates.thor | 78 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 lib/tasks/updates.thor diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index 083b0015..89191e48 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -132,6 +132,35 @@ module Docs end end + def get_latest_version + raise NotImplementedError + end + + # Returns whether or not this scraper is outdated. + # + # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions. + # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases. + # + # Scrapers of documentations that do not use this versioning approach should override this method. + # + # Examples of the default implementation: + # 1 -> 2 = outdated + # 1.1 -> 1.2 = outdated + # 1.1.1 -> 1.1.2 = not outdated + def is_outdated(current_version, latest_version) + current_parts = current_version.split(/\./).map(&:to_i) + latest_parts = latest_version.split(/\./).map(&:to_i) + + # Only check the first two parts, the third part is for patch updates + [0, 1].each do |i| + break if i >= current_parts.length or i >= latest_parts.length + return true if latest_parts[i] > current_parts[i] + return false if latest_parts[i] < current_parts[i] + end + + false + end + private def request_one(url) diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor new file mode 100644 index 00000000..35d52e28 --- /dev/null +++ b/lib/tasks/updates.thor @@ -0,0 +1,78 @@ +class UpdatesCLI < Thor + def self.to_s + 'Updates' + end + + def initialize(*args) + require 'docs' + require 'progress_bar' + super + end + + desc 'check [doc]...', 'Check for outdated documentations' + def check(*names) + # Convert names to a list of Scraper instances + # Versions are omitted, if v10 is outdated than v8 is aswell + docs = names.map {|name| Docs.find(name.split(/@|~/)[0], false)}.uniq + + # Check all documentations for updates when no arguments are given + docs = Docs.all if docs.empty? + + progress_bar = ::ProgressBar.new docs.length + progress_bar.write + + results = docs.map do |doc| + result = check_doc(doc) + progress_bar.increment! + result + end + + outdated = results.select {|result| result.is_a?(Hash) && result[:is_outdated]} + return if outdated.empty? + + logger.info("Outdated documentations (#{outdated.length}):") + outdated.each do |result| + logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}") + end + rescue Docs::DocNotFound => error + logger.error(error) + logger.info('Run "thor docs:list" to see the list of docs.') + end + + private + + def check_doc(doc) + # Scraper versions are always sorted from new to old + # Therefore, the first item's release value is the latest current scraper version + # + # For example, a scraper could scrape 3 versions: 10, 11 and 12 + # doc.versions.first would be the scraper for version 12 if the scraper is written like all the other scrapers are + instance = doc.versions.first.new + + current_version = instance.options[:release] + return nil if current_version.nil? + + latest_version = instance.get_latest_version + return nil if latest_version.nil? + + { + name: doc.name, + current_version: current_version, + latest_version: latest_version, + is_outdated: instance.is_outdated(current_version, latest_version) + } + rescue NotImplementedError + logger.warn("Can't check #{doc.name}, get_latest_version is not implemented") + rescue => error + logger.error("Error while checking #{doc.name}: #{error}") + end + + def logger + @logger ||= Logger.new($stdout).tap do |logger| + logger.formatter = proc do |severity, datetime, progname, msg| + prefix = severity != "INFO" ? "[#{severity}] " : "" + "#{prefix}#{msg}\n" + end + end + end +end From 2d1e8aa00cce957358cb741f180c3ff8dabfbefa Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Fri, 8 Mar 2019 15:13:11 +0100 Subject: [PATCH 02/17] Add better logging and get_latest_version implementations for 10 scrapers --- lib/docs/core/scraper.rb | 34 +++++++++++++++++++- lib/docs/scrapers/angular.rb | 4 +++ lib/docs/scrapers/angularjs.rb | 4 +++ lib/docs/scrapers/ansible.rb | 6 ++++ lib/docs/scrapers/apache.rb | 6 ++++ lib/docs/scrapers/apache_pig.rb | 6 ++++ lib/docs/scrapers/async.rb | 7 ++++ lib/docs/scrapers/babel.rb | 6 ++++ lib/docs/scrapers/backbone.rb | 7 ++++ lib/docs/scrapers/bash.rb | 7 ++++ lib/docs/scrapers/bluebird.rb | 4 +++ lib/docs/scrapers/pygame.rb | 1 + lib/tasks/updates.thor | 57 +++++++++++++++++++++------------ 13 files changed, 128 insertions(+), 21 deletions(-) diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index 89191e48..a7e388a8 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -132,7 +132,7 @@ module Docs end end - def get_latest_version + def get_latest_version(&block) raise NotImplementedError end @@ -231,6 +231,38 @@ module Docs {} end + # Utility methods for get_latest_version + + def fetch(url, &block) + Request.run(url) do |response| + if response.success? + block.call response.body + else + block.call nil + end + end + end + + def fetch_doc(url, &block) + fetch(url) do |body| + parser = Parser.new(body) + block.call parser.html + end + end + + def fetch_json(url, &block) + fetch(url) do |body| + json = JSON.parse(body) + block.call json + end + end + + def get_npm_version(package, &block) + fetch_json("https://registry.npmjs.com/#{package}") do |json| + block.call json['dist-tags']['latest'] + end + end + module FixInternalUrlsBehavior def self.included(base) base.extend ClassMethods diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index c318ce25..fa03eb36 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -155,6 +155,10 @@ module Docs end end + def get_latest_version(&block) + get_npm_version('@angular/core', &block) + end + private def parse(response) diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb index b8ff08b9..aa74ca1c 100644 --- a/lib/docs/scrapers/angularjs.rb +++ b/lib/docs/scrapers/angularjs.rb @@ -69,5 +69,9 @@ module Docs self.release = '1.2.32' self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end + + def get_latest_version(&block) + get_npm_version('angular', &block) + end end end diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb index 2d62909a..60fb1953 100644 --- a/lib/docs/scrapers/ansible.rb +++ b/lib/docs/scrapers/ansible.rb @@ -87,5 +87,11 @@ module Docs quickstart.html list_of_all_modules.html) end + + def get_latest_version(&block) + fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc| + block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip + end + end end end diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb index 9ee82f12..5eca041e 100644 --- a/lib/docs/scrapers/apache.rb +++ b/lib/docs/scrapers/apache.rb @@ -33,5 +33,11 @@ module Docs © 2018 The Apache Software Foundation
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(&block) + fetch_doc('http://httpd.apache.org/docs/') do |doc| + block.call doc.at_css('#apcontents > ul a')['href'][0...-1] + end + end end end diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb index 65897a78..15c477bf 100644 --- a/lib/docs/scrapers/apache_pig.rb +++ b/lib/docs/scrapers/apache_pig.rb @@ -43,5 +43,11 @@ module Docs self.base_url = "https://pig.apache.org/docs/r#{release}/" end + def get_latest_version(&block) + fetch_doc('https://pig.apache.org/') do |doc| + item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') + block.call item.content.strip.sub(/Release /, '') + end + end end end diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 40022f19..930820b4 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -17,5 +17,12 @@ module Docs © 2010–2018 Caolan McMahon
Licensed under the MIT License. HTML + + def get_latest_version(&block) + fetch_doc('https://caolan.github.io/async/') do |doc| + version = doc.at_css('#version-dropdown > a').content.strip[1..-1] + block.call version + end + end end end diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb index c9e40212..cc8bec6d 100644 --- a/lib/docs/scrapers/babel.rb +++ b/lib/docs/scrapers/babel.rb @@ -22,5 +22,11 @@ module Docs stub '' do '
' end + + def get_latest_version(&block) + fetch_doc('https://babeljs.io/docs/en/') do |doc| + block.call doc.at_css('a[href="/versions"] > h3').content + end + end end end diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb index b72b1084..2fb7662f 100644 --- a/lib/docs/scrapers/backbone.rb +++ b/lib/docs/scrapers/backbone.rb @@ -20,5 +20,12 @@ module Docs © 2010–2016 Jeremy Ashkenas, DocumentCloud
Licensed under the MIT License. HTML + + def get_latest_version(&block) + fetch_doc('https://backbonejs.org/') do |doc| + version = doc.at_css('.version').content + block.call version[1...-1] + end + end end end diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb index feb0ddce..b62868a6 100644 --- a/lib/docs/scrapers/bash.rb +++ b/lib/docs/scrapers/bash.rb @@ -17,5 +17,12 @@ module Docs Copyright © 2000, 2001, 2002, 2007, 2008 Free Software Foundation, Inc.
Licensed under the GNU Free Documentation License. HTML + + def get_latest_version(&block) + fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body| + version = body.scan(/, Version ([0-9.]+)/)[0][0] + block.call version[0...-1] + end + end end end diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb index e5cd6b59..73888004 100644 --- a/lib/docs/scrapers/bluebird.rb +++ b/lib/docs/scrapers/bluebird.rb @@ -18,5 +18,9 @@ module Docs © 2013–2017 Petka Antonov
Licensed under the MIT License. HTML + + def get_latest_version(&block) + get_npm_version('bluebird', &block) + end end end diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb index 9da3148d..892619e4 100644 --- a/lib/docs/scrapers/pygame.rb +++ b/lib/docs/scrapers/pygame.rb @@ -2,6 +2,7 @@ module Docs class Pygame < UrlScraper self.type = 'simple' self.release = '1.9.4' + self.base_url = 'https://www.pygame.org/docs/' self.root_path = 'py-modindex.html' self.links = { home: 'https://www.pygame.org/', diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index 35d52e28..eb3467f2 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -9,7 +9,8 @@ class UpdatesCLI < Thor super end - desc 'check [doc]...', 'Check for outdated documentations' + desc 'check [--verbose] [doc]...', 'Check for outdated documentations' + option :verbose, :type => :boolean def check(*names) # Convert names to a list of Scraper instances # Versions are omitted, if v10 is outdated than v8 is aswell @@ -27,13 +28,14 @@ class UpdatesCLI < Thor result end - outdated = results.select {|result| result.is_a?(Hash) && result[:is_outdated]} - return if outdated.empty? + valid_results = results.select {|result| result.is_a?(Hash)} - logger.info("Outdated documentations (#{outdated.length}):") - outdated.each do |result| - logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}") - end + up_to_date_results = valid_results.select {|result| !result[:is_outdated]} + outdated_results = valid_results.select {|result| result[:is_outdated]} + + log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty? + logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty? + log_results('Outdated', outdated_results) unless outdated_results.empty? rescue Docs::DocNotFound => error logger.error(error) logger.info('Run "thor docs:list" to see the list of docs.') @@ -42,33 +44,48 @@ class UpdatesCLI < Thor private def check_doc(doc) - # Scraper versions are always sorted from new to old + # Newer scraper versions always come before older scraper versions # Therefore, the first item's release value is the latest current scraper version # # For example, a scraper could scrape 3 versions: 10, 11 and 12 - # doc.versions.first would be the scraper for version 12 if the scraper is written like all the other scrapers are + # doc.versions.first would be the scraper for version 12 instance = doc.versions.first.new + return nil unless instance.class.method_defined?(:options) + current_version = instance.options[:release] return nil if current_version.nil? - latest_version = instance.get_latest_version - return nil if latest_version.nil? + logger.debug("Checking #{doc.name}") + + instance.get_latest_version do |latest_version| + return { + name: doc.name, + current_version: current_version, + latest_version: latest_version, + is_outdated: instance.is_outdated(current_version, latest_version) + } + end - { - name: doc.name, - current_version: current_version, - latest_version: latest_version, - is_outdated: instance.is_outdated(current_version, latest_version) - } + return nil rescue NotImplementedError - logger.warn("Can't check #{doc.name}, get_latest_version is not implemented") - rescue => error - logger.error("Error while checking #{doc.name}: #{error}") + logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") + rescue + logger.error("Error while checking #{doc.name}") + raise + end + + def log_results(label, results) + logger.info("#{label} documentations (#{results.length}):") + + results.each do |result| + logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}") + end end def logger @logger ||= Logger.new($stdout).tap do |logger| + logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO logger.formatter = proc do |severity, datetime, progname, msg| prefix = severity != "INFO" ? "[#{severity}] " : "" "#{prefix}#{msg}\n" From 3dc17a9b29a33bb77f8b22ae5b9e8cf8ea9fd8f9 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sat, 9 Mar 2019 02:36:05 +0100 Subject: [PATCH 03/17] Finish get_latest_version for 81 scrapers and add uploading functionality --- Gemfile | 1 + Gemfile.lock | 3 + lib/docs/core/scraper.rb | 56 +++-- lib/docs/scrapers/angular.rb | 4 +- lib/docs/scrapers/angularjs.rb | 4 +- lib/docs/scrapers/ansible.rb | 4 +- lib/docs/scrapers/apache.rb | 4 +- lib/docs/scrapers/apache_pig.rb | 4 +- lib/docs/scrapers/async.rb | 4 +- lib/docs/scrapers/babel.rb | 4 +- lib/docs/scrapers/backbone.rb | 4 +- lib/docs/scrapers/bash.rb | 4 +- lib/docs/scrapers/bluebird.rb | 4 +- lib/docs/scrapers/bootstrap.rb | 6 + lib/docs/scrapers/bottle.rb | 7 + lib/docs/scrapers/bower.rb | 4 + lib/docs/scrapers/cakephp.rb | 6 + lib/docs/scrapers/chai.rb | 4 + lib/docs/scrapers/chef.rb | 7 + lib/docs/scrapers/clojure.rb | 6 + lib/docs/scrapers/cmake.rb | 7 + lib/docs/scrapers/codeception.rb | 6 + lib/docs/scrapers/codeceptjs.rb | 4 + lib/docs/scrapers/codeigniter.rb | 7 + lib/docs/scrapers/coffeescript.rb | 4 + lib/docs/scrapers/cordova.rb | 9 + lib/docs/scrapers/crystal.rb | 6 + lib/docs/scrapers/d.rb | 6 + lib/docs/scrapers/d3.rb | 4 + lib/docs/scrapers/dart.rb | 7 + lib/docs/scrapers/django.rb | 6 + lib/docs/scrapers/docker.rb | 7 + lib/docs/scrapers/dojo.rb | 6 + lib/docs/scrapers/drupal.rb | 9 + lib/docs/scrapers/electron.rb | 6 + lib/docs/scrapers/elixir.rb | 6 + lib/docs/scrapers/ember.rb | 6 + lib/docs/scrapers/erlang.rb | 6 + lib/docs/scrapers/eslint.rb | 4 + lib/docs/scrapers/express.rb | 4 + lib/docs/scrapers/falcon.rb | 6 + lib/docs/scrapers/fish.rb | 6 + lib/docs/scrapers/flow.rb | 4 + lib/docs/scrapers/git.rb | 6 + lib/docs/scrapers/gnu/gcc.rb | 7 + lib/docs/scrapers/gnu/gnu_fortran.rb | 7 + lib/docs/scrapers/go.rb | 9 + lib/docs/scrapers/godot.rb | 6 + lib/docs/scrapers/graphite.rb | 6 + lib/docs/scrapers/grunt.rb | 4 + lib/docs/scrapers/handlebars.rb | 4 + lib/docs/scrapers/haskell.rb | 7 + lib/docs/scrapers/haxe.rb | 7 + lib/docs/scrapers/homebrew.rb | 6 + lib/docs/scrapers/immutable.rb | 4 + lib/docs/scrapers/influxdata.rb | 7 + lib/docs/scrapers/jasmine.rb | 6 + lib/docs/scrapers/jekyll.rb | 6 + lib/docs/scrapers/jest.rb | 6 + lib/docs/scrapers/jquery/jquery_core.rb | 4 + lib/docs/scrapers/jquery/jquery_mobile.rb | 7 + lib/docs/scrapers/jquery/jquery_ui.rb | 4 + lib/docs/scrapers/jsdoc.rb | 6 + lib/docs/scrapers/julia.rb | 6 + lib/docs/scrapers/knockout.rb | 6 + lib/docs/scrapers/koa.rb | 4 + lib/docs/scrapers/kotlin.rb | 6 + lib/docs/scrapers/laravel.rb | 6 + lib/docs/scrapers/leaflet.rb | 6 + lib/docs/scrapers/less.rb | 7 + lib/docs/scrapers/liquid.rb | 6 + lib/docs/scrapers/lodash.rb | 6 + lib/docs/scrapers/love.rb | 6 + lib/docs/scrapers/lua.rb | 6 + lib/docs/scrapers/marionette.rb | 4 + lib/docs/scrapers/matplotlib.rb | 6 + lib/docs/scrapers/meteor.rb | 6 + lib/docs/scrapers/mocha.rb | 4 + lib/docs/scrapers/modernizr.rb | 4 + lib/docs/scrapers/moment.rb | 6 + lib/docs/scrapers/mongoose.rb | 7 + lib/docs/scrapers/rdoc/minitest.rb | 6 + lib/docs/scrapers/rdoc/rails.rb | 6 + lib/docs/scrapers/rdoc/ruby.rb | 12 + lib/tasks/updates.thor | 286 +++++++++++++++++++--- 85 files changed, 739 insertions(+), 68 deletions(-) diff --git a/Gemfile b/Gemfile index 31b57064..5b8fae70 100644 --- a/Gemfile +++ b/Gemfile @@ -40,6 +40,7 @@ group :docs do gem 'unix_utils', require: false gem 'tty-pager', require: false gem 'net-sftp', '>= 2.1.3.rc2', require: false + gem 'terminal-table', require: false end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index d968a27b..3ed8570b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,6 +101,8 @@ GEM unicode-display_width (~> 1.4.0) unicode_utils (~> 1.4.0) strings-ansi (0.1.0) + terminal-table (1.8.0) + unicode-display_width (~> 1.1, >= 1.1.1) thin (1.7.2) daemons (~> 1.0, >= 1.0.9) eventmachine (~> 1.0, >= 1.0.4) @@ -153,6 +155,7 @@ DEPENDENCIES sinatra-contrib sprockets sprockets-helpers + terminal-table thin thor tty-pager diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index a7e388a8..b124c6db 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -132,7 +132,7 @@ module Docs end end - def get_latest_version(&block) + def get_latest_version(options, &block) raise NotImplementedError end @@ -147,15 +147,15 @@ module Docs # 1 -> 2 = outdated # 1.1 -> 1.2 = outdated # 1.1.1 -> 1.1.2 = not outdated - def is_outdated(current_version, latest_version) - current_parts = current_version.split(/\./).map(&:to_i) + def is_outdated(scraper_version, latest_version) + scraper_parts = scraper_version.split(/\./).map(&:to_i) latest_parts = latest_version.split(/\./).map(&:to_i) # Only check the first two parts, the third part is for patch updates [0, 1].each do |i| - break if i >= current_parts.length or i >= latest_parts.length - return true if latest_parts[i] > current_parts[i] - return false if latest_parts[i] < current_parts[i] + break if i >= scraper_parts.length or i >= latest_parts.length + return true if latest_parts[i] > scraper_parts[i] + return false if latest_parts[i] < scraper_parts[i] end false @@ -231,38 +231,62 @@ module Docs {} end + # # Utility methods for get_latest_version + # - def fetch(url, &block) - Request.run(url) do |response| + def fetch(url, options, &block) + headers = {} + + if options.key?(:github_token) and url.start_with?('https://api.github.com/') + headers['Authorization'] = "token #{options[:github_token]}" + end + + options[:logger].debug("Fetching #{url}") + + Request.run(url, { headers: headers }) do |response| if response.success? block.call response.body else + options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") block.call nil end end end - def fetch_doc(url, &block) - fetch(url) do |body| - parser = Parser.new(body) - block.call parser.html + def fetch_doc(url, options, &block) + fetch(url, options) do |body| + block.call Nokogiri::HTML.parse body, nil, 'UTF-8' end end - def fetch_json(url, &block) - fetch(url) do |body| + def fetch_json(url, options, &block) + fetch(url, options) do |body| json = JSON.parse(body) block.call json end end - def get_npm_version(package, &block) - fetch_json("https://registry.npmjs.com/#{package}") do |json| + def get_npm_version(package, options, &block) + fetch_json("https://registry.npmjs.com/#{package}", options) do |json| block.call json['dist-tags']['latest'] end end + def get_latest_github_release(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) + end + + def get_github_tags(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) + end + + def get_github_file_contents(owner, repo, path, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| + block.call(Base64.decode64(json['content'])) + end + end + module FixInternalUrlsBehavior def self.included(base) base.extend ClassMethods diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index fa03eb36..059b0e8e 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -155,8 +155,8 @@ module Docs end end - def get_latest_version(&block) - get_npm_version('@angular/core', &block) + def get_latest_version(options, &block) + get_npm_version('@angular/core', options, &block) end private diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb index aa74ca1c..b6e18325 100644 --- a/lib/docs/scrapers/angularjs.rb +++ b/lib/docs/scrapers/angularjs.rb @@ -70,8 +70,8 @@ module Docs self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end - def get_latest_version(&block) - get_npm_version('angular', &block) + def get_latest_version(options, &block) + get_npm_version('angular', options, &block) end end end diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb index 60fb1953..293f74a7 100644 --- a/lib/docs/scrapers/ansible.rb +++ b/lib/docs/scrapers/ansible.rb @@ -88,8 +88,8 @@ module Docs list_of_all_modules.html) end - def get_latest_version(&block) - fetch_doc('https://docs.ansible.com/ansible/latest/index.html') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc| block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip end end diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb index 5eca041e..ba0fa340 100644 --- a/lib/docs/scrapers/apache.rb +++ b/lib/docs/scrapers/apache.rb @@ -34,8 +34,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(&block) - fetch_doc('http://httpd.apache.org/docs/') do |doc| + def get_latest_version(options, &block) + fetch_doc('http://httpd.apache.org/docs/', options) do |doc| block.call doc.at_css('#apcontents > ul a')['href'][0...-1] end end diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb index 15c477bf..5454140b 100644 --- a/lib/docs/scrapers/apache_pig.rb +++ b/lib/docs/scrapers/apache_pig.rb @@ -43,8 +43,8 @@ module Docs self.base_url = "https://pig.apache.org/docs/r#{release}/" end - def get_latest_version(&block) - fetch_doc('https://pig.apache.org/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://pig.apache.org/', options) do |doc| item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') block.call item.content.strip.sub(/Release /, '') end diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 930820b4..18e9bbbf 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -18,8 +18,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - fetch_doc('https://caolan.github.io/async/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://caolan.github.io/async/', options) do |doc| version = doc.at_css('#version-dropdown > a').content.strip[1..-1] block.call version end diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb index cc8bec6d..675f86be 100644 --- a/lib/docs/scrapers/babel.rb +++ b/lib/docs/scrapers/babel.rb @@ -23,8 +23,8 @@ module Docs '
' end - def get_latest_version(&block) - fetch_doc('https://babeljs.io/docs/en/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://babeljs.io/docs/en/', options) do |doc| block.call doc.at_css('a[href="/versions"] > h3').content end end diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb index 2fb7662f..ad6220e5 100644 --- a/lib/docs/scrapers/backbone.rb +++ b/lib/docs/scrapers/backbone.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - fetch_doc('https://backbonejs.org/') do |doc| + def get_latest_version(options, &block) + fetch_doc('https://backbonejs.org/', options) do |doc| version = doc.at_css('.version').content block.call version[1...-1] end diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb index b62868a6..5556f5b9 100644 --- a/lib/docs/scrapers/bash.rb +++ b/lib/docs/scrapers/bash.rb @@ -18,8 +18,8 @@ module Docs Licensed under the GNU Free Documentation License. HTML - def get_latest_version(&block) - fetch('https://www.gnu.org/software/bash/manual/html_node/index.html') do |body| + def get_latest_version(options, &block) + fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body| version = body.scan(/, Version ([0-9.]+)/)[0][0] block.call version[0...-1] end diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb index 73888004..8a960b87 100644 --- a/lib/docs/scrapers/bluebird.rb +++ b/lib/docs/scrapers/bluebird.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(&block) - get_npm_version('bluebird', &block) + def get_latest_version(options, &block) + get_npm_version('bluebird', options, &block) end end end diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb index 7b2406b8..aa0b4cc3 100644 --- a/lib/docs/scrapers/bootstrap.rb +++ b/lib/docs/scrapers/bootstrap.rb @@ -34,5 +34,11 @@ module Docs options[:only] = %w(getting-started/ css/ components/ javascript/) end + + def get_latest_version(options, &block) + fetch_doc('https://getbootstrap.com/', options) do |doc| + block.call doc.at_css('#bd-versions').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/bottle.rb b/lib/docs/scrapers/bottle.rb index 25ad7f6e..6e4a19a8 100644 --- a/lib/docs/scrapers/bottle.rb +++ b/lib/docs/scrapers/bottle.rb @@ -27,5 +27,12 @@ module Docs self.release = '0.11.7' self.base_url = "https://bottlepy.org/docs/#{self.version}/" end + + def get_latest_version(options, &block) + fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc| + label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') + block.call label.content.sub(/Bottle /, '') + end + end end end diff --git a/lib/docs/scrapers/bower.rb b/lib/docs/scrapers/bower.rb index b032f1d3..1102ee75 100644 --- a/lib/docs/scrapers/bower.rb +++ b/lib/docs/scrapers/bower.rb @@ -19,5 +19,9 @@ module Docs © 2018 Bower contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('bower', options, &block) + end end end diff --git a/lib/docs/scrapers/cakephp.rb b/lib/docs/scrapers/cakephp.rb index 08dbead0..b123ab7a 100644 --- a/lib/docs/scrapers/cakephp.rb +++ b/lib/docs/scrapers/cakephp.rb @@ -71,6 +71,12 @@ module Docs self.base_url = 'https://api.cakephp.org/2.7/' end + def get_latest_version(options, &block) + fetch_doc('https://api.cakephp.org/3.7/', options) do |doc| + block.call doc.at_css('.version-picker .dropdown-toggle').content.strip + end + end + private def parse(response) diff --git a/lib/docs/scrapers/chai.rb b/lib/docs/scrapers/chai.rb index 9d8aa4d2..422bd5a9 100644 --- a/lib/docs/scrapers/chai.rb +++ b/lib/docs/scrapers/chai.rb @@ -23,5 +23,9 @@ module Docs © 2016 Chai.js Assertion Library
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('chai', options, &block) + end end end diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb index 2fd32a83..337d1202 100644 --- a/lib/docs/scrapers/chef.rb +++ b/lib/docs/scrapers/chef.rb @@ -47,5 +47,12 @@ module Docs options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//] end + + def get_latest_version(options, &block) + fetch_doc('https://docs-archive.chef.io/', options) do |doc| + cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)') + block.call cell.content.sub(/Chef Client /, '') + end + end end end diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb index c6bdcdea..465a4493 100644 --- a/lib/docs/scrapers/clojure.rb +++ b/lib/docs/scrapers/clojure.rb @@ -27,5 +27,11 @@ module Docs self.release = '1.7' self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/' end + + def get_latest_version(options, &block) + fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc| + block.call doc.at_css('#header-version').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb index c455e4fd..dde4721c 100644 --- a/lib/docs/scrapers/cmake.rb +++ b/lib/docs/scrapers/cmake.rb @@ -59,5 +59,12 @@ module Docs self.release = '3.5.2' self.base_url = 'https://cmake.org/cmake/help/v3.5/' end + + def get_latest_version(options, &block) + fetch_doc('https://cmake.org/documentation/', options) do |doc| + link = doc.at_css('.entry-content ul > li > strong > a > big') + block.call link.content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb index 919f146d..2e28de7f 100644 --- a/lib/docs/scrapers/codeception.rb +++ b/lib/docs/scrapers/codeception.rb @@ -18,5 +18,11 @@ module Docs © 2011 Michael Bodnarchuk and contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://codeception.com/changelog', options) do |doc| + block.call doc.at_css('#page > h4').content + end + end end end diff --git a/lib/docs/scrapers/codeceptjs.rb b/lib/docs/scrapers/codeceptjs.rb index 13189340..e3f4fda8 100644 --- a/lib/docs/scrapers/codeceptjs.rb +++ b/lib/docs/scrapers/codeceptjs.rb @@ -21,5 +21,9 @@ module Docs © 2015 DavertMik <davert@codegyre.com> (http://codegyre.com)
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('codeceptjs', options, &block) + end end end diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb index 573f9b8c..864cf700 100644 --- a/lib/docs/scrapers/codeigniter.rb +++ b/lib/docs/scrapers/codeigniter.rb @@ -38,5 +38,12 @@ module Docs version '3' do self.release = '3.1.8' end + + def get_latest_version(options, &block) + fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc| + header = doc.at_css('#change-log h2') + block.call header.content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb index 23e9557f..d848d208 100644 --- a/lib/docs/scrapers/coffeescript.rb +++ b/lib/docs/scrapers/coffeescript.rb @@ -30,5 +30,9 @@ module Docs options[:container] = '.container' end + + def get_latest_version(options, &block) + get_npm_version('coffeescript', options, &block) + end end end diff --git a/lib/docs/scrapers/cordova.rb b/lib/docs/scrapers/cordova.rb index f74c72ff..efe8fb03 100644 --- a/lib/docs/scrapers/cordova.rb +++ b/lib/docs/scrapers/cordova.rb @@ -42,5 +42,14 @@ module Docs self.release = '6.5.0' self.base_url = 'https://cordova.apache.org/docs/en/6.x/' end + + def get_latest_version(options, &block) + fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc| + label = doc.at_css('#versionDropdown').content.strip + version = label.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end end end diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb index 29061a1d..e70317f2 100644 --- a/lib/docs/scrapers/crystal.rb +++ b/lib/docs/scrapers/crystal.rb @@ -34,5 +34,11 @@ module Docs HTML end } + + def get_latest_version(options, &block) + fetch('https://crystal-lang.org/api', options) do |body| + block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/d.rb b/lib/docs/scrapers/d.rb index 6126380e..b0adaf31 100644 --- a/lib/docs/scrapers/d.rb +++ b/lib/docs/scrapers/d.rb @@ -26,5 +26,11 @@ module Docs def initial_urls %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html) end + + def get_latest_version(options, &block) + fetch_doc('https://dlang.org/changelog/', options) do |doc| + block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id'] + end + end end end diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb index 26b27ca5..cfbbafc9 100644 --- a/lib/docs/scrapers/d3.rb +++ b/lib/docs/scrapers/d3.rb @@ -58,5 +58,9 @@ module Docs options[:root_title] = 'D3.js' options[:only_patterns] = [/\.md\z/] end + + def get_latest_version(options, &block) + get_npm_version('d3', options, &block) + end end end diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb index c345c22f..42d20423 100644 --- a/lib/docs/scrapers/dart.rb +++ b/lib/docs/scrapers/dart.rb @@ -31,5 +31,12 @@ module Docs self.release = '1.24.3' self.base_url = "https://api.dartlang.org/stable/#{release}/" end + + def get_latest_version(options, &block) + fetch_doc('https://api.dartlang.org/', options) do |doc| + label = doc.at_css('footer > span').content.strip + block.call label.sub(/Dart /, '') + end + end end end diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb index 45273540..746c0f40 100644 --- a/lib/docs/scrapers/django.rb +++ b/lib/docs/scrapers/django.rb @@ -63,5 +63,11 @@ module Docs self.release = '1.8.18' self.base_url = 'https://docs.djangoproject.com/en/1.8/' end + + def get_latest_version(options, &block) + fetch_doc('https://docs.djangoproject.com/', options) do |doc| + block.call doc.at_css('#doc-versions > li.current > span > strong').content + end + end end end diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb index 92494f8a..dd849391 100644 --- a/lib/docs/scrapers/docker.rb +++ b/lib/docs/scrapers/docker.rb @@ -137,5 +137,12 @@ module Docs options[:container] = '#docs' options[:only_patterns] << /\Aswarm\// end + + def get_latest_version(options, &block) + fetch_doc('https://docs.docker.com/', options) do |doc| + label = doc.at_css('.nav-container button.dropdown-toggle').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb index 937ed21a..66dccb6f 100644 --- a/lib/docs/scrapers/dojo.rb +++ b/lib/docs/scrapers/dojo.rb @@ -36,6 +36,12 @@ module Docs urls.map { |url| "#{url}" }.join end + def get_latest_version(options, &block) + fetch_doc('https://dojotoolkit.org/api/', options) do |doc| + block.call doc.at_css('#versionSelector > option[selected]').content + end + end + private def get_url_list(json, set = Set.new) diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb index 5710eb36..92da4193 100644 --- a/lib/docs/scrapers/drupal.rb +++ b/lib/docs/scrapers/drupal.rb @@ -98,5 +98,14 @@ module Docs /\A[\w\-\.]+\.php\/7\.x\z/ ] end + + def get_latest_version(options, &block) + fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc| + version = doc.at_css('td.form > form > select > option[selected]').content + version = version.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end end end diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb index 3cb399f0..dd3cf00a 100644 --- a/lib/docs/scrapers/electron.rb +++ b/lib/docs/scrapers/electron.rb @@ -22,5 +22,11 @@ module Docs © 2013–2018 GitHub Inc.
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://electronjs.org/docs', options) do |doc| + block.call doc.at_css('.docs-version').content + end + end end end diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index 10d5aac1..d5b8dbe6 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -97,5 +97,11 @@ module Docs 'https://elixir-lang.org/getting-started/' ] end + + def get_latest_version(options, &block) + fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc| + block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/ember.rb b/lib/docs/scrapers/ember.rb index 3db20c94..24a8817e 100644 --- a/lib/docs/scrapers/ember.rb +++ b/lib/docs/scrapers/ember.rb @@ -56,5 +56,11 @@ module Docs https://emberjs.com/api/ember-data/2.14/classes/DS ) end + + def get_latest_version(options, &block) + fetch_doc('https://emberjs.com/api/ember/release', options) do |doc| + block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip + end + end end end diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb index d6aa2a0b..7dcb0fae 100644 --- a/lib/docs/scrapers/erlang.rb +++ b/lib/docs/scrapers/erlang.rb @@ -55,5 +55,11 @@ module Docs version '18' do self.release = '18.3' end + + def get_latest_version(options, &block) + fetch_doc('https://www.erlang.org/downloads', options) do |doc| + block.call doc.at_css('.col-lg-3 > ul > li').content.strip + end + end end end diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb index 8b4c9a2e..dac9c283 100644 --- a/lib/docs/scrapers/eslint.rb +++ b/lib/docs/scrapers/eslint.rb @@ -20,5 +20,9 @@ module Docs © JS Foundation and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('eslint', options, &block) + end end end diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb index 0fb4ed14..67ba07e8 100644 --- a/lib/docs/scrapers/express.rb +++ b/lib/docs/scrapers/express.rb @@ -28,5 +28,9 @@ module Docs © 2017 StrongLoop, IBM, and other expressjs.com contributors.
Licensed under the Creative Commons Attribution-ShareAlike License v3.0. HTML + + def get_latest_version(options, &block) + get_npm_version('express', options, &block) + end end end diff --git a/lib/docs/scrapers/falcon.rb b/lib/docs/scrapers/falcon.rb index 5bfd8efc..cd5b70cd 100644 --- a/lib/docs/scrapers/falcon.rb +++ b/lib/docs/scrapers/falcon.rb @@ -33,5 +33,11 @@ module Docs self.release = '1.2.0' self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/" end + + def get_latest_version(options, &block) + fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc| + block.call doc.at_css('#changelogs ul > li > a').content + end + end end end diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb index 5ccfa71c..9340961a 100644 --- a/lib/docs/scrapers/fish.rb +++ b/lib/docs/scrapers/fish.rb @@ -46,5 +46,11 @@ module Docs self.release = '2.2.0' self.base_url = "https://fishshell.com/docs/#{version}/" end + + def get_latest_version(options, &block) + fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc| + block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb index 16ea70dd..546473f7 100644 --- a/lib/docs/scrapers/flow.rb +++ b/lib/docs/scrapers/flow.rb @@ -18,5 +18,9 @@ module Docs © 2013–present Facebook Inc.
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('flow-bin', options, &block) + end end end diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb index 26b2da95..f10473d0 100644 --- a/lib/docs/scrapers/git.rb +++ b/lib/docs/scrapers/git.rb @@ -19,5 +19,11 @@ module Docs © 2005–2018 Linus Torvalds and others
Licensed under the GNU General Public License version 2. HTML + + def get_latest_version(options, &block) + fetch_doc('https://git-scm.com/', options) do |doc| + block.call doc.at_css('.version').content.strip + end + end end end diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb index be3bb54e..3252dd6d 100644 --- a/lib/docs/scrapers/gnu/gcc.rb +++ b/lib/docs/scrapers/gnu/gcc.rb @@ -99,5 +99,12 @@ module Docs options[:replace_paths] = CPP_PATHS end + + def get_latest_version(options, &block) + fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| + label = doc.at_css('ul > li > ul > li > a').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb index 2610178e..f72f7d65 100644 --- a/lib/docs/scrapers/gnu/gnu_fortran.rb +++ b/lib/docs/scrapers/gnu/gnu_fortran.rb @@ -25,5 +25,12 @@ module Docs self.release = '4.9.3' self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/" end + + def get_latest_version(options, &block) + fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| + label = doc.at_css('ul > li > ul > li > a').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb index 7b233317..6f8f7a4a 100644 --- a/lib/docs/scrapers/go.rb +++ b/lib/docs/scrapers/go.rb @@ -24,6 +24,15 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML + def get_latest_version(options, &block) + fetch_doc('https://golang.org/pkg/', options) do |doc| + footer = doc.at_css('#footer').content + version = footer.scan(/go([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + block.call version + end + end + private def parse(response) # Hook here because Nokogori removes whitespace from textareas diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb index 7e7da9a6..d43782c2 100644 --- a/lib/docs/scrapers/godot.rb +++ b/lib/docs/scrapers/godot.rb @@ -37,5 +37,11 @@ module Docs self.release = '2.1' self.base_url = "http://docs.godotengine.org/en/#{self.version}/" end + + def get_latest_version(options, &block) + fetch_doc('https://docs.godotengine.org/', options) do |doc| + block.call doc.at_css('.version').content.strip + end + end end end diff --git a/lib/docs/scrapers/graphite.rb b/lib/docs/scrapers/graphite.rb index 49ade898..d1d8b9d1 100644 --- a/lib/docs/scrapers/graphite.rb +++ b/lib/docs/scrapers/graphite.rb @@ -17,5 +17,11 @@ module Docs © 2011–2016 The Graphite Project
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(options, &block) + fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc| + block.call doc.at_css('#release-notes li > a').content + end + end end end diff --git a/lib/docs/scrapers/grunt.rb b/lib/docs/scrapers/grunt.rb index 2201c043..1e8af9fb 100644 --- a/lib/docs/scrapers/grunt.rb +++ b/lib/docs/scrapers/grunt.rb @@ -26,5 +26,9 @@ module Docs © GruntJS Team
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('grunt-cli', options, &block) + end end end diff --git a/lib/docs/scrapers/handlebars.rb b/lib/docs/scrapers/handlebars.rb index 22935d21..7df63102 100644 --- a/lib/docs/scrapers/handlebars.rb +++ b/lib/docs/scrapers/handlebars.rb @@ -19,5 +19,9 @@ module Docs © 2011–2017 by Yehuda Katz
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('handlebars', options, &block) + end end end diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index 442339b3..383e1990 100755 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -68,5 +68,12 @@ module Docs options[:only_patterns] = [/\Alibraries\//] end + + def get_latest_version(options, &block) + fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc| + label = doc.at_css('.related > ul > li:last-child').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/haxe.rb b/lib/docs/scrapers/haxe.rb index 33f20b93..5a685efc 100644 --- a/lib/docs/scrapers/haxe.rb +++ b/lib/docs/scrapers/haxe.rb @@ -66,5 +66,12 @@ module Docs version 'Python' do self.base_url = 'https://api.haxe.org/python/' end + + def get_latest_version(options, &block) + fetch_doc('https://api.haxe.org/', options) do |doc| + label = doc.at_css('.container.main-content h1 > small').content + block.call label.sub(/version /, '') + end + end end end diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index fba79ec0..fef1ed05 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -19,5 +19,11 @@ module Docs © 2009–present Homebrew contributors
Licensed under the BSD 2-Clause License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('Homebrew', 'brew', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb index fa7fb81b..342ce107 100644 --- a/lib/docs/scrapers/immutable.rb +++ b/lib/docs/scrapers/immutable.rb @@ -54,5 +54,9 @@ module Docs JS capybara.html end + + def get_latest_version(options, &block) + get_npm_version('immutable', options, &block) + end end end diff --git a/lib/docs/scrapers/influxdata.rb b/lib/docs/scrapers/influxdata.rb index 6c83b66b..4fc98c16 100644 --- a/lib/docs/scrapers/influxdata.rb +++ b/lib/docs/scrapers/influxdata.rb @@ -46,5 +46,12 @@ module Docs © 2015 InfluxData, Inc.
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc| + label = doc.at_css('.navbar--current-product').content.strip + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb index 82f3c9cf..5f38e3d5 100644 --- a/lib/docs/scrapers/jasmine.rb +++ b/lib/docs/scrapers/jasmine.rb @@ -17,5 +17,11 @@ module Docs © 2008–2017 Pivotal Labs
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('jasmine', 'jasmine', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/jekyll.rb b/lib/docs/scrapers/jekyll.rb index 1faaa9de..a6af352f 100644 --- a/lib/docs/scrapers/jekyll.rb +++ b/lib/docs/scrapers/jekyll.rb @@ -28,5 +28,11 @@ module Docs © 2008–2018 Tom Preston-Werner and Jekyll contributors
Licensed under the MIT license. HTML + + def get_latest_version(options, &block) + fetch_doc('https://jekyllrb.com/docs/', options) do |doc| + block.call doc.at_css('.meta a').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb index f4ce944f..71efcf54 100644 --- a/lib/docs/scrapers/jest.rb +++ b/lib/docs/scrapers/jest.rb @@ -17,5 +17,11 @@ module Docs © 2014–present Facebook Inc.
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc| + block.call doc.at_css('header > a > h3').content + end + end end end diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb index 20aca0dc..dad609e7 100644 --- a/lib/docs/scrapers/jquery/jquery_core.rb +++ b/lib/docs/scrapers/jquery/jquery_core.rb @@ -22,5 +22,9 @@ module Docs /Selectors\/odd/i, /index/i ] + + def get_latest_version(options, &block) + get_npm_version('jquery', options, &block) + end end end diff --git a/lib/docs/scrapers/jquery/jquery_mobile.rb b/lib/docs/scrapers/jquery/jquery_mobile.rb index 8e5abf1c..53b2c624 100644 --- a/lib/docs/scrapers/jquery/jquery_mobile.rb +++ b/lib/docs/scrapers/jquery/jquery_mobile.rb @@ -16,5 +16,12 @@ module Docs options[:fix_urls] = ->(url) do url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/' end + + def get_latest_version(options, &block) + fetch_doc('https://jquerymobile.com/', options) do |doc| + label = doc.at_css('.download-box > .download-option:last-child > span').content + block.call label.sub(/Version /, '') + end + end end end diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb index 0c90fc1a..05c276e1 100644 --- a/lib/docs/scrapers/jquery/jquery_ui.rb +++ b/lib/docs/scrapers/jquery/jquery_ui.rb @@ -15,5 +15,9 @@ module Docs options[:fix_urls] = ->(url) do url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/' end + + def get_latest_version(options, &block) + get_npm_version('jquery-ui', options, &block) + end end end diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb index bb3781ca..39feca71 100644 --- a/lib/docs/scrapers/jsdoc.rb +++ b/lib/docs/scrapers/jsdoc.rb @@ -21,5 +21,11 @@ module Docs © 2011–2017 the contributors to the JSDoc 3 documentation project
Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML + + def get_latest_version(options, &block) + get_latest_github_release('jsdoc3', 'jsdoc', options) do |release| + block.call release['tag_name'] + end + end end end diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb index 5bc16b77..0875835a 100644 --- a/lib/docs/scrapers/julia.rb +++ b/lib/docs/scrapers/julia.rb @@ -49,5 +49,11 @@ module Docs html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html' end + + def get_latest_version(options, &block) + get_latest_github_release('JuliaLang', 'julia', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb index 663f7847..60af1540 100644 --- a/lib/docs/scrapers/knockout.rb +++ b/lib/docs/scrapers/knockout.rb @@ -33,5 +33,11 @@ module Docs © Steven Sanderson, the Knockout.js team, and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('knockout', 'knockout', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/koa.rb b/lib/docs/scrapers/koa.rb index 3ce79cac..4d90e30f 100644 --- a/lib/docs/scrapers/koa.rb +++ b/lib/docs/scrapers/koa.rb @@ -34,5 +34,9 @@ module Docs © 2018 Koa contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('koa', options, &block) + end end end diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb index 415393d1..7539212d 100644 --- a/lib/docs/scrapers/kotlin.rb +++ b/lib/docs/scrapers/kotlin.rb @@ -28,5 +28,11 @@ module Docs © 2010–2018 JetBrains s.r.o.
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(options, &block) + get_latest_github_release('JetBrains', 'kotlin', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index 5c88ae0f..cdf32732 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -133,5 +133,11 @@ module Docs url end end + + def get_latest_version(options, &block) + get_latest_github_release('laravel', 'laravel', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb index c8e2071c..24bc6142 100644 --- a/lib/docs/scrapers/leaflet.rb +++ b/lib/docs/scrapers/leaflet.rb @@ -39,5 +39,11 @@ module Docs self.base_url = "https://leafletjs.com/reference-#{release}.html" end + def get_latest_version(options, &block) + fetch_doc('https://leafletjs.com/index.html', options) do |doc| + link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first + block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/less.rb b/lib/docs/scrapers/less.rb index a0947e1a..00c884eb 100644 --- a/lib/docs/scrapers/less.rb +++ b/lib/docs/scrapers/less.rb @@ -21,5 +21,12 @@ module Docs © 2009–2016 The Core Less Team
Licensed under the Creative Commons Attribution License 3.0. HTML + + def get_latest_version(options, &block) + fetch_doc('http://lesscss.org/features/', options) do |doc| + label = doc.at_css('.footer-links > li').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/liquid.rb b/lib/docs/scrapers/liquid.rb index 9ebc4041..4630b2d1 100644 --- a/lib/docs/scrapers/liquid.rb +++ b/lib/docs/scrapers/liquid.rb @@ -19,5 +19,11 @@ module Docs © 2005, 2006 Tobias Luetke
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_github_tags('Shopify', 'liquid', options) do |tags| + block.call tags[0]['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb index 0461f7b7..5488b9ab 100644 --- a/lib/docs/scrapers/lodash.rb +++ b/lib/docs/scrapers/lodash.rb @@ -32,5 +32,11 @@ module Docs self.release = '2.4.2' self.base_url = "https://lodash.com/docs/#{release}" end + + def get_latest_version(options, &block) + fetch_doc('https://lodash.com/docs/', options) do |doc| + block.call doc.at_css('#version > option[selected]').content + end + end end end diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb index 7f23bded..019edbab 100644 --- a/lib/docs/scrapers/love.rb +++ b/lib/docs/scrapers/love.rb @@ -39,5 +39,11 @@ module Docs © 2006–2016 LÖVE Development Team
Licensed under the GNU Free Documentation License, Version 1.3. HTML + + def get_latest_version(options, &block) + fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc| + block.call doc.at_css('#mw-content-text table a').content + end + end end end diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb index 40a5c007..30af5523 100644 --- a/lib/docs/scrapers/lua.rb +++ b/lib/docs/scrapers/lua.rb @@ -26,5 +26,11 @@ module Docs self.release = '5.1.5' self.base_url = 'https://www.lua.org/manual/5.1/' end + + def get_latest_version(options, &block) + fetch_doc('https://www.lua.org/manual/', options) do |doc| + block.call doc.at_css('p.menubar > a').content + end + end end end diff --git a/lib/docs/scrapers/marionette.rb b/lib/docs/scrapers/marionette.rb index fea6617f..12de6d0c 100644 --- a/lib/docs/scrapers/marionette.rb +++ b/lib/docs/scrapers/marionette.rb @@ -38,5 +38,9 @@ module Docs html_filters.push 'marionette/entries_v2' end + + def get_latest_version(options, &block) + get_npm_version('backbone.marionette', options, &block) + end end end diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb index ddd1f9de..948955a6 100644 --- a/lib/docs/scrapers/matplotlib.rb +++ b/lib/docs/scrapers/matplotlib.rb @@ -64,5 +64,11 @@ module Docs "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/" ] end + + def get_latest_version(options, &block) + get_latest_github_release('matplotlib', 'matplotlib', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/meteor.rb b/lib/docs/scrapers/meteor.rb index b38d5dc2..02b81bc3 100644 --- a/lib/docs/scrapers/meteor.rb +++ b/lib/docs/scrapers/meteor.rb @@ -45,5 +45,11 @@ module Docs self.base_urls = ['https://guide.meteor.com/v1.3/', "https://docs.meteor.com/v#{self.release}/"] options[:fix_urls] = nil end + + def get_latest_version(options, &block) + fetch_doc('https://docs.meteor.com/#/full/', options) do |doc| + block.call doc.at_css('select.version-select > option').content + end + end end end diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb index 8ab9bdc8..6654d754 100644 --- a/lib/docs/scrapers/mocha.rb +++ b/lib/docs/scrapers/mocha.rb @@ -18,5 +18,9 @@ module Docs © 2011–2018 JS Foundation and contributors
Licensed under the Creative Commons Attribution 4.0 International License. HTML + + def get_latest_version(options, &block) + get_npm_version('mocha', options, &block) + end end end diff --git a/lib/docs/scrapers/modernizr.rb b/lib/docs/scrapers/modernizr.rb index 96c82153..93a738bb 100644 --- a/lib/docs/scrapers/modernizr.rb +++ b/lib/docs/scrapers/modernizr.rb @@ -15,5 +15,9 @@ module Docs © 2009–2017 The Modernizr team
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('modernizr', options, &block) + end end end diff --git a/lib/docs/scrapers/moment.rb b/lib/docs/scrapers/moment.rb index 88df0d14..9dd27107 100644 --- a/lib/docs/scrapers/moment.rb +++ b/lib/docs/scrapers/moment.rb @@ -22,5 +22,11 @@ module Docs © JS Foundation and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('http://momentjs.com/', options) do |doc| + block.call doc.at_css('.hero-title > h1 > span').content + end + end end end diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb index 71ee04d2..2d221990 100644 --- a/lib/docs/scrapers/mongoose.rb +++ b/lib/docs/scrapers/mongoose.rb @@ -26,5 +26,12 @@ module Docs © 2010 LearnBoost
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://mongoosejs.com/docs/', options) do |doc| + label = doc.at_css('.pure-menu-link').content.strip + block.call label.sub(/Version /, '') + end + end end end diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb index 761da1de..f676010d 100644 --- a/lib/docs/scrapers/rdoc/minitest.rb +++ b/lib/docs/scrapers/rdoc/minitest.rb @@ -21,5 +21,11 @@ module Docs © Ryan Davis, seattle.rb
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents| + block.call contents.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb index 6bdce34a..9cb2ab9b 100644 --- a/lib/docs/scrapers/rdoc/rails.rb +++ b/lib/docs/scrapers/rdoc/rails.rb @@ -93,5 +93,11 @@ module Docs version '4.1' do self.release = '4.1.16' end + + def get_latest_version(options, &block) + get_latest_github_release('rails', 'rails', options) do |release| + block.call release['name'] + end + end end end diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb index dd296765..292540db 100644 --- a/lib/docs/scrapers/rdoc/ruby.rb +++ b/lib/docs/scrapers/rdoc/ruby.rb @@ -84,5 +84,17 @@ module Docs version '2.2' do self.release = '2.2.10' end + + def get_latest_version(options, &block) + get_github_tags('ruby', 'ruby', options) do |tags| + tags.each do |tag| + version = tag['name'].gsub(/_/, '.')[1..-1] + if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 + block.call version + break + end + end + end + end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index eb3467f2..f370544c 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -1,4 +1,12 @@ class UpdatesCLI < Thor + # The GitHub user that is allowed to upload reports + # TODO: Update this before creating a PR + UPLOAD_USER = 'jmerle' + + # The repository to create an issue in when uploading the results + # TODO: Update this before creating a PR + UPLOAD_REPO = 'jmerle/devdocs' + def self.to_s 'Updates' end @@ -6,10 +14,14 @@ class UpdatesCLI < Thor def initialize(*args) require 'docs' require 'progress_bar' + require 'terminal-table' + require 'date' super end - desc 'check [--verbose] [doc]...', 'Check for outdated documentations' + desc 'check [--github-token] [--upload] [--verbose] [doc]...', 'Check for outdated documentations' + option :github_token, :type => :string + option :upload, :type => :boolean option :verbose, :type => :boolean def check(*names) # Convert names to a list of Scraper instances @@ -19,23 +31,26 @@ class UpdatesCLI < Thor # Check all documentations for updates when no arguments are given docs = Docs.all if docs.empty? - progress_bar = ::ProgressBar.new docs.length - progress_bar.write + opts = { + logger: logger + } - results = docs.map do |doc| - result = check_doc(doc) - progress_bar.increment! - result + if options.key?(:github_token) + opts[:github_token] = options[:github_token] end - valid_results = results.select {|result| result.is_a?(Hash)} + with_progress_bar do |bar| + bar.max = docs.length + bar.write + end - up_to_date_results = valid_results.select {|result| !result[:is_outdated]} - outdated_results = valid_results.select {|result| result[:is_outdated]} + results = docs.map do |doc| + result = check_doc(doc, opts) + with_progress_bar(&:increment!) + result + end - log_results('Up-to-date', up_to_date_results) if options[:verbose] and !up_to_date_results.empty? - logger.info("") if options[:verbose] and !up_to_date_results.empty? and !outdated_results.empty? - log_results('Outdated', outdated_results) unless outdated_results.empty? + process_results(results) rescue Docs::DocNotFound => error logger.error(error) logger.info('Run "thor docs:list" to see the list of docs.') @@ -43,53 +58,260 @@ class UpdatesCLI < Thor private - def check_doc(doc) + def check_doc(doc, options) # Newer scraper versions always come before older scraper versions - # Therefore, the first item's release value is the latest current scraper version + # Therefore, the first item's release value is the latest scraper version # # For example, a scraper could scrape 3 versions: 10, 11 and 12 # doc.versions.first would be the scraper for version 12 instance = doc.versions.first.new - return nil unless instance.class.method_defined?(:options) - - current_version = instance.options[:release] - return nil if current_version.nil? + scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil + return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil? logger.debug("Checking #{doc.name}") - instance.get_latest_version do |latest_version| + instance.get_latest_version(options) do |latest_version| return { name: doc.name, - current_version: current_version, + scraper_version: scraper_version, latest_version: latest_version, - is_outdated: instance.is_outdated(current_version, latest_version) + is_outdated: instance.is_outdated(scraper_version, latest_version) } end - - return nil rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") + error_result(doc, '`get_latest_version` is not implemented') rescue logger.error("Error while checking #{doc.name}") raise end - def log_results(label, results) - logger.info("#{label} documentations (#{results.length}):") + def error_result(doc, reason) + { + name: doc.name, + error: reason + } + end + + def process_results(results) + successful_results = results.select {|result| result.key?(:is_outdated)} + failed_results = results.select {|result| result.key?(:error)} + + up_to_date_results = successful_results.select {|result| !result[:is_outdated]} + outdated_results = successful_results.select {|result| result[:is_outdated]} + + log_results(outdated_results, up_to_date_results, failed_results) + upload_results(outdated_results, up_to_date_results, failed_results) if options[:upload] + end + + # + # Result logging methods + # + + def log_results(outdated_results, up_to_date_results, failed_results) + log_failed_results(failed_results) unless failed_results.empty? + log_successful_results('Up-to-date', up_to_date_results) unless up_to_date_results.empty? + log_successful_results('Outdated', outdated_results) unless outdated_results.empty? + end + + def log_successful_results(label, results) + title = "#{label} documentations (#{results.length})" + headings = ['Documentation', 'Scraper version', 'Latest version'] + rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]} + + table = Terminal::Table.new :title => title, :headings => headings, :rows => rows + puts table + end + + def log_failed_results(results) + title = "Documentations that could not be checked (#{results.length})" + headings = %w(Documentation Reason) + rows = results.map {|result| [result[:name], result[:error]]} + + table = Terminal::Table.new :title => title, :headings => headings, :rows => rows + puts table + end + + # + # Upload methods + # + + def upload_results(outdated_results, up_to_date_results, failed_results) + # We can't create issues without a GitHub token + unless options.key?(:github_token) + logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the --github-token parameter') + return + end + + logger.info('Uploading the results to a new GitHub issue') + + logger.info('Checking if the GitHub token belongs to the correct user') + github_get('/user') do |user| + # Only allow the DevDocs bot to upload reports + if user['login'] == UPLOAD_USER + issue = results_to_issue(outdated_results, up_to_date_results, failed_results) + + logger.info('Creating a new GitHub issue') + github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue| + search_params = { + q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", + sort: 'created', + order: 'desc' + } + + logger.info('Checking if the previous issue is still open') + github_get('/search/issues', search_params) do |matching_issues| + previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} + + if previous_issue.nil? + logger.info('No previous issue found') + log_upload_success(created_issue) + else + comment = "This report was superseded by ##{created_issue['number']}." + + logger.info('Commenting on the previous issue') + github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_| + if previous_issue['closed_at'].nil? + logger.info('Closing the previous issue') + github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_| + log_upload_success(created_issue) + end + else + logger.info('The previous issue has already been closed') + log_upload_success(created_issue) + end + end + end + end + end + else + logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + end + end + end + + def results_to_issue(outdated_results, up_to_date_results, failed_results) + results = [ + successful_results_to_markdown('Outdated', outdated_results), + successful_results_to_markdown('Up-to-date', up_to_date_results), + failed_results_to_markdown(failed_results) + ] + + results_str = results.select {|result| !result.nil?}.join("\n\n") + + title = "Documentation versions report for #{Date.today.strftime('%B')} 2019" + body = <<-MARKDOWN +## What is this? + +This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations. + +Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created. + +## Results + +The #{outdated_results.length + up_to_date_results.length + failed_results.length} documentations are divided as follows: +- #{outdated_results.length} that #{outdated_results.length == 1 ? 'is' : 'are'} outdated +- #{up_to_date_results.length} that #{up_to_date_results.length == 1 ? 'is' : 'are'} up-to-date (patch updates are ignored) +- #{failed_results.length} that could not be checked + MARKDOWN + + { + title: title, + body: body.strip + "\n\n" + results_str + } + end + + def successful_results_to_markdown(label, results) + return nil if results.empty? + + title = "#{label} documentations (#{results.length})" + headings = ['Documentation', 'Scraper version', 'Latest version'] + rows = results.map {|result| [result[:name], result[:scraper_version], result[:latest_version]]} + + results_to_markdown(title, headings, rows) + end + + def failed_results_to_markdown(results) + return nil if results.empty? + + title = "Documentations that could not be checked (#{results.length})" + headings = %w(Documentation Reason) + rows = results.map {|result| [result[:name], result[:error]]} + + results_to_markdown(title, headings, rows) + end + + def results_to_markdown(title, headings, rows) + "
\n#{title}\n\n#{create_markdown_table(headings, rows)}\n
" + end + + def create_markdown_table(headings, rows) + header = headings.join(' | ') + separator = '-|' * headings.length + body = rows.map {|row| row.join(' | ')} + + header + "\n" + separator[0...-1] + "\n" + body.join("\n") + end + + def log_upload_success(created_issue) + logger.info("Successfully uploaded the results to #{created_issue['html_url']}") + end + + # + # HTTP utilities + # + + def github_get(endpoint, params = {}, &block) + github_request(endpoint, {method: :get, params: params}, &block) + end + + def github_post(endpoint, params, &block) + github_request(endpoint, {method: :post, body: params.to_json}, &block) + end + + def github_patch(endpoint, params, &block) + github_request(endpoint, {method: :patch, body: params.to_json}, &block) + end - results.each do |result| - logger.info("#{result[:name]}: #{result[:current_version]} -> #{result[:latest_version]}") + def github_request(endpoint, opts, &block) + url = "https://api.github.com#{endpoint}" + + # GitHub token authentication + opts[:headers] = { + Authorization: "token #{options[:github_token]}" + } + + # GitHub requires the Content-Type to be application/json when a body is passed + if opts.key?(:body) + opts[:headers]['Content-Type'] = 'application/json' end + + logger.debug("Making a #{opts[:method]} request to #{url}") + + Docs::Request.run(url, opts) do |response| + # response.success? is false if the response code is 201 + # GitHub returns 201 Created after an issue is created + if response.success? || response.code == 201 + block.call JSON.parse(response.body) + else + logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") + block.call nil + end + end + end + + # A utility method which ensures no progress bar is shown when stdout is not a tty + def with_progress_bar(&block) + return unless $stdout.tty? + @progress_bar ||= ::ProgressBar.new + block.call @progress_bar end def logger @logger ||= Logger.new($stdout).tap do |logger| logger.level = options[:verbose] ? Logger::DEBUG : Logger::INFO - logger.formatter = proc do |severity, datetime, progname, msg| - prefix = severity != "INFO" ? "[#{severity}] " : "" - "#{prefix}#{msg}\n" - end + logger.formatter = proc {|severity, datetime, progname, msg| "[#{severity}] #{msg}\n"} end end end From b8a7965bfebea5c432832d8f7f3a5ee6a9cea39b Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sat, 9 Mar 2019 02:50:06 +0100 Subject: [PATCH 04/17] Automatically check and upload on Travis CI cron job --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d5e5a6aa..e8d3d588 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,3 +5,7 @@ cache: bundler before_script: - gem update --system - gem install bundler + +script: + - if [ "$TRAVIS_EVENT_TYPE" != "cron" ]; then bundle exec rake; fi + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then bundle exec thor updates:check --github-token $GH_TOKEN --upload; fi From de6e42fa09239ff3c90998c807b619987ed4407e Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sat, 9 Mar 2019 03:13:47 +0100 Subject: [PATCH 05/17] Update documentation --- docs/Scraper-Reference.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/adding-docs.md | 1 + 2 files changed, 40 insertions(+) diff --git a/docs/Scraper-Reference.md b/docs/Scraper-Reference.md index d9fe8b4a..60d377d8 100644 --- a/docs/Scraper-Reference.md +++ b/docs/Scraper-Reference.md @@ -184,3 +184,42 @@ More information about how filters work is available on the [Filter Reference](. Overrides the `:title` option for the root page only. _Note: this filter is disabled by default._ + +## Keeping scrapers up-to-date + +In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden by all scrapers that define the `self.release` attribute. This method should return the latest version of the documentation that is being scraped. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. + +To make life easier, there are a few utility methods that you can use in `get_latest_version`: +* `fetch(url, options, &block)` + + Makes a GET request to the url and calls `&block` with the body. + + Example: [lib/docs/scrapers/bash.rb](../lib/docs/scrapers/bash.rb) +* `fetch_doc(url, options, &block)` + + Makes a GET request to the url and calls `&block` with the HTML body converted to a Nokogiri document. + + Example: [lib/docs/scrapers/git.rb](../lib/docs/scrapers/git.rb) +* `fetch_json(url, options, &block)` + + Makes a GET request to the url and calls `&block` with the JSON body converted to a dictionary. +* `get_npm_version(package, options, &block)` + + Calls `&block` with the latest version of the given npm package. + + Example: [lib/docs/scrapers/bower.rb](../lib/docs/scrapers/bower.rb) +* `get_latest_github_release(owner, repo, options, &block)` + + Calls `&block` with the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). + + Example: [lib/docs/scrapers/jsdoc.rb](../lib/docs/scrapers/jsdoc.rb) +* `get_github_tags(owner, repo, options, &block)` + + Calls `&block` with the list of tags on the given repository ([format](https://developer.github.com/v3/repos/#list-tags)). + + Example: [lib/docs/scrapers/liquid.rb](../lib/docs/scrapers/liquid.rb) +* `get_github_file_contents(owner, repo, path, options, &block)` + + Calls `&block` with the contents of the requested file in the default branch of the given repository. + + Example: [lib/docs/scrapers/minitest.rb](../lib/docs/scrapers/minitest.rb) diff --git a/docs/adding-docs.md b/docs/adding-docs.md index 03c6b87a..baafc59a 100644 --- a/docs/adding-docs.md +++ b/docs/adding-docs.md @@ -16,6 +16,7 @@ Adding a documentation may look like a daunting task but once you get the hang o 9. To customize the pages' styling, create an SCSS file in the `assets/stylesheets/pages/` directory and import it in both `application.css.scss` AND `application-dark.css.scss`. Both the file and CSS class should be named `_[type]` where [type] is equal to the scraper's `type` attribute (documentations with the same type share the same custom CSS and JS). _(Note: feel free to submit a pull request without custom CSS/JS)_ 10. To add syntax highlighting or execute custom JavaScript on the pages, create a file in the `assets/javascripts/views/pages/` directory (take a look at the other files to see how it works). 11. Add the documentation's icon in the `public/icons/docs/[my_doc]/` directory, in both 16x16 and 32x32-pixels formats. It'll be added to the icon sprite after your pull request is merged. +12. Ensure `thor updates:check [my_doc]` shows the correct latest version. If the documentation includes more than a few hundreds pages and is available for download, try to scrape it locally (e.g. using `FileScraper`). It'll make the development process much faster and avoids putting too much load on the source site. (It's not a problem if your scraper is coupled to your local setup, just explain how it works in your pull request.) From 0315c88f81b82e648ecc5aab8d51015f836a8d47 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sat, 9 Mar 2019 03:28:33 +0100 Subject: [PATCH 06/17] Fix Erlang get_latest_version implementation --- lib/docs/scrapers/erlang.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb index 7dcb0fae..0211da55 100644 --- a/lib/docs/scrapers/erlang.rb +++ b/lib/docs/scrapers/erlang.rb @@ -58,7 +58,7 @@ module Docs def get_latest_version(options, &block) fetch_doc('https://www.erlang.org/downloads', options) do |doc| - block.call doc.at_css('.col-lg-3 > ul > li').content.strip + block.call doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '') end end end From 6ee1693134bbef44912a3a53cf250c6f82e98fbc Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sat, 9 Mar 2019 20:22:48 +0100 Subject: [PATCH 07/17] Implement get_latest_version for the remaining scrapers --- lib/docs/scrapers/nginx.rb | 7 +++++++ lib/docs/scrapers/nginx_lua_module.rb | 10 ++++++++++ lib/docs/scrapers/nim.rb | 6 ++++++ lib/docs/scrapers/node.rb | 6 ++++++ lib/docs/scrapers/nokogiri2.rb | 6 ++++++ lib/docs/scrapers/npm.rb | 6 ++++++ lib/docs/scrapers/numpy.rb | 6 ++++++ lib/docs/scrapers/openjdk.rb | 21 +++++++++++++++++++++ lib/docs/scrapers/opentsdb.rb | 6 ++++++ lib/docs/scrapers/padrino.rb | 6 ++++++ lib/docs/scrapers/pandas.rb | 7 +++++++ lib/docs/scrapers/perl.rb | 6 ++++++ lib/docs/scrapers/phalcon.rb | 6 ++++++ lib/docs/scrapers/phaser.rb | 6 ++++++ lib/docs/scrapers/phoenix.rb | 6 ++++++ lib/docs/scrapers/php.rb | 7 +++++++ lib/docs/scrapers/phpunit.rb | 7 +++++++ lib/docs/scrapers/postgresql.rb | 7 +++++++ lib/docs/scrapers/pug.rb | 4 ++++ lib/docs/scrapers/puppeteer.rb | 6 ++++++ lib/docs/scrapers/pygame.rb | 6 ++++++ lib/docs/scrapers/python.rb | 6 ++++++ lib/docs/scrapers/q.rb | 4 ++++ lib/docs/scrapers/qt.rb | 7 +++++++ lib/docs/scrapers/ramda.rb | 6 ++++++ lib/docs/scrapers/react.rb | 6 ++++++ lib/docs/scrapers/react_native.rb | 6 ++++++ lib/docs/scrapers/redis.rb | 7 +++++++ lib/docs/scrapers/redux.rb | 4 ++++ lib/docs/scrapers/relay.rb | 6 ++++++ lib/docs/scrapers/requirejs.rb | 4 ++++ lib/docs/scrapers/rethinkdb.rb | 6 ++++++ lib/docs/scrapers/rust.rb | 7 +++++++ lib/docs/scrapers/sass.rb | 6 ++++++ lib/docs/scrapers/scikit_image.rb | 6 ++++++ lib/docs/scrapers/scikit_learn.rb | 6 ++++++ lib/docs/scrapers/sinon.rb | 6 ++++++ lib/docs/scrapers/socketio.rb | 4 ++++ lib/docs/scrapers/sqlite.rb | 6 ++++++ lib/docs/scrapers/statsmodels.rb | 5 +++++ lib/docs/scrapers/symfony.rb | 6 ++++++ lib/docs/scrapers/tcl_tk.rb | 7 +++++++ lib/docs/scrapers/tensorflow.rb | 6 ++++++ lib/docs/scrapers/terraform.rb | 6 ++++++ lib/docs/scrapers/twig.rb | 6 ++++++ lib/docs/scrapers/typescript.rb | 6 ++++++ lib/docs/scrapers/underscore.rb | 6 ++++++ lib/docs/scrapers/vagrant.rb | 6 ++++++ lib/docs/scrapers/vue.rb | 6 ++++++ lib/docs/scrapers/vulkan.rb | 6 ++++++ lib/docs/scrapers/webpack.rb | 4 ++++ lib/docs/scrapers/yarn.rb | 6 ++++++ lib/docs/scrapers/yii.rb | 6 ++++++ lib/tasks/updates.thor | 4 ++-- 54 files changed, 335 insertions(+), 2 deletions(-) diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb index 5a3dbf1e..07a4565f 100644 --- a/lib/docs/scrapers/nginx.rb +++ b/lib/docs/scrapers/nginx.rb @@ -25,5 +25,12 @@ module Docs © 2011-2018 Nginx, Inc.
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://nginx.org/en/download.html', options) do |doc| + table = doc.at_css('#content > table').inner_html + block.call table.scan(/nginx-([0-9.]+) Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + get_github_tags('openresty', 'lua-nginx-module', options) do |tags| + tag = tags.find { |tag| !tag['name'].include?('rc') } + block.call tag['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/nim.rb b/lib/docs/scrapers/nim.rb index cce16f1c..28d83049 100644 --- a/lib/docs/scrapers/nim.rb +++ b/lib/docs/scrapers/nim.rb @@ -17,5 +17,11 @@ module Docs © 2006–2018 Andreas Rumpf
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://nim-lang.org/docs/overview.html', options) do |doc| + block.call doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip + end + end end end diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb index 66c6a456..36c14e9e 100644 --- a/lib/docs/scrapers/node.rb +++ b/lib/docs/scrapers/node.rb @@ -46,5 +46,11 @@ module Docs self.release = '4.9.1' self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/' end + + def get_latest_version(options, &block) + fetch_doc('https://nodejs.org/en/', options) do |doc| + block.call doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb index 084ad2fb..aca5d10c 100644 --- a/lib/docs/scrapers/nokogiri2.rb +++ b/lib/docs/scrapers/nokogiri2.rb @@ -19,5 +19,11 @@ module Docs Patrick Mahoney, Yoko Harada, Akinori Musha, John Shahid, Lars Kanis
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('sparklemotion', 'nokogiri', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb index 2c854e6e..7ef727c0 100644 --- a/lib/docs/scrapers/npm.rb +++ b/lib/docs/scrapers/npm.rb @@ -29,5 +29,11 @@ module Docs Licensed under the npm License.
npm is a trademark of npm, Inc. HTML + + def get_latest_version(options, &block) + get_latest_github_release('npm', 'cli', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb index 1327fc02..1fcc05d6 100644 --- a/lib/docs/scrapers/numpy.rb +++ b/lib/docs/scrapers/numpy.rb @@ -49,5 +49,11 @@ module Docs self.release = '1.10.4' self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/" end + + def get_latest_version(options, &block) + get_latest_github_release('numpy', 'numpy', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb index 944ac416..e833c7c4 100644 --- a/lib/docs/scrapers/openjdk.rb +++ b/lib/docs/scrapers/openjdk.rb @@ -86,5 +86,26 @@ module Docs def read_file(path) File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil end + + def get_latest_version(options, &block) + latest_version = 8 + current_attempt = latest_version + attempts = 0 + + while attempts < 3 + current_attempt += 1 + + fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", options) do |doc| + if doc.at_css('.perror').nil? + latest_version = current_attempt + attempts = 0 + else + attempts += 1 + end + end + end + + block.call latest_version.to_s + end end end diff --git a/lib/docs/scrapers/opentsdb.rb b/lib/docs/scrapers/opentsdb.rb index 0372e331..54a77017 100644 --- a/lib/docs/scrapers/opentsdb.rb +++ b/lib/docs/scrapers/opentsdb.rb @@ -18,5 +18,11 @@ module Docs © 2010–2016 The OpenTSDB Authors
Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses. HTML + + def get_latest_version(options, &block) + get_latest_github_release('OpenTSDB', 'opentsdb', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/padrino.rb b/lib/docs/scrapers/padrino.rb index 218f1731..78588ef4 100644 --- a/lib/docs/scrapers/padrino.rb +++ b/lib/docs/scrapers/padrino.rb @@ -23,5 +23,11 @@ module Docs stub 'index2' do request_one(url_for('index')).body end + + def get_latest_version(options, &block) + get_github_tags('padrino', 'padrino-framework', options) do |tags| + block.call tags[0]['name'] + end + end end end diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb index 1355a012..f50c4427 100644 --- a/lib/docs/scrapers/pandas.rb +++ b/lib/docs/scrapers/pandas.rb @@ -49,5 +49,12 @@ module Docs self.release = '0.18.1' self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/" end + + def get_latest_version(options, &block) + fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', options) do |doc| + label = doc.at_css('.body > .section > p').content + block.call label.scan(/Version: ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb index 142ceaa5..136a6d75 100644 --- a/lib/docs/scrapers/perl.rb +++ b/lib/docs/scrapers/perl.rb @@ -43,5 +43,11 @@ module Docs self.release = '5.20.2' self.base_url = "https://perldoc.perl.org/#{self.release}/" end + + def get_latest_version(options, &block) + fetch('https://perldoc.perl.org/static/perlversion.js', options) do |body| + block.call body.scan(/>Perl ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/phalcon.rb b/lib/docs/scrapers/phalcon.rb index c2cb9242..82a9cf85 100644 --- a/lib/docs/scrapers/phalcon.rb +++ b/lib/docs/scrapers/phalcon.rb @@ -29,5 +29,11 @@ module Docs self.release = '2.0.13' self.base_url = 'https://docs.phalconphp.com/en/2.0.0/' end + + def get_latest_version(options, &block) + fetch_doc('https://docs.phalconphp.com/', options) do |doc| + block.call doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') + end + end end end diff --git a/lib/docs/scrapers/phaser.rb b/lib/docs/scrapers/phaser.rb index 5ae371c1..ef265e2e 100644 --- a/lib/docs/scrapers/phaser.rb +++ b/lib/docs/scrapers/phaser.rb @@ -25,5 +25,11 @@ module Docs © 2016 Richard Davey, Photon Storm Ltd.
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('photonstorm', 'phaser', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/phoenix.rb b/lib/docs/scrapers/phoenix.rb index 26b0144f..08948b3c 100644 --- a/lib/docs/scrapers/phoenix.rb +++ b/lib/docs/scrapers/phoenix.rb @@ -46,5 +46,11 @@ module Docs HTML end } + + def get_latest_version(options, &block) + fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', options) do |doc| + block.call doc.at_css('.sidebar-projectVersion').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb index d4a66b5b..b6aac039 100644 --- a/lib/docs/scrapers/php.rb +++ b/lib/docs/scrapers/php.rb @@ -66,5 +66,12 @@ module Docs © 1997–2018 The PHP Documentation Group
Licensed under the Creative Commons Attribution License v3.0 or later. HTML + + def get_latest_version(options, &block) + fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', options) do |doc| + label = doc.at_css('tbody.gen-changelog > tr > td').content + block.call label.split(',').last.strip + end + end end end diff --git a/lib/docs/scrapers/phpunit.rb b/lib/docs/scrapers/phpunit.rb index 665ca4c7..c5e8c396 100644 --- a/lib/docs/scrapers/phpunit.rb +++ b/lib/docs/scrapers/phpunit.rb @@ -37,5 +37,12 @@ module Docs self.release = '4.8' self.base_url = "https://phpunit.de/manual/#{release}/en/" end + + def get_latest_version(options, &block) + fetch_doc('https://phpunit.readthedocs.io/', options) do |doc| + label = doc.at_css('.rst-current-version').content.strip + block.call label.scan(/v: ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb index 109685ba..4946066b 100644 --- a/lib/docs/scrapers/postgresql.rb +++ b/lib/docs/scrapers/postgresql.rb @@ -80,5 +80,12 @@ module Docs html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names' end + + def get_latest_version(options, &block) + fetch_doc('https://www.postgresql.org/docs/current/index.html', options) do |doc| + label = doc.at_css('#pgContentWrap h1.title').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/pug.rb b/lib/docs/scrapers/pug.rb index 76949529..9084feb1 100644 --- a/lib/docs/scrapers/pug.rb +++ b/lib/docs/scrapers/pug.rb @@ -18,6 +18,10 @@ module Docs Licensed under the MIT license. HTML + def get_latest_version(options, &block) + get_npm_version('pug', options, &block) + end + private def parse(response) # Hook here because Nokogori removes whitespace from textareas diff --git a/lib/docs/scrapers/puppeteer.rb b/lib/docs/scrapers/puppeteer.rb index 2e26c180..a1506c7a 100644 --- a/lib/docs/scrapers/puppeteer.rb +++ b/lib/docs/scrapers/puppeteer.rb @@ -14,5 +14,11 @@ module Docs © 2017 Google Inc
Licensed under the Apache License 2.0. HTML + + def get_latest_version(options, &block) + get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', options) do |contents| + block.call contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb index 892619e4..538c0722 100644 --- a/lib/docs/scrapers/pygame.rb +++ b/lib/docs/scrapers/pygame.rb @@ -17,5 +17,11 @@ module Docs © Pygame Developpers.
Licensed under the GNU LGPL License version 2.1. HTML + + def get_latest_version(options, &block) + get_latest_github_release('pygame', 'pygame', options) do |release| + block.call release['tag_name'] + end + end end end diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb index aa4336d9..16ecc366 100644 --- a/lib/docs/scrapers/python.rb +++ b/lib/docs/scrapers/python.rb @@ -50,5 +50,11 @@ module Docs html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html' end + + def get_latest_version(options, &block) + fetch_doc('https://docs.python.org/', options) do |doc| + block.call doc.at_css('.version_switcher_placeholder').content + end + end end end diff --git a/lib/docs/scrapers/q.rb b/lib/docs/scrapers/q.rb index bb66c156..0a160488 100644 --- a/lib/docs/scrapers/q.rb +++ b/lib/docs/scrapers/q.rb @@ -19,5 +19,9 @@ module Docs © 2009–2017 Kristopher Michael Kowal
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('q', options, &block) + end end end diff --git a/lib/docs/scrapers/qt.rb b/lib/docs/scrapers/qt.rb index 412eca6d..41c835e9 100644 --- a/lib/docs/scrapers/qt.rb +++ b/lib/docs/scrapers/qt.rb @@ -117,5 +117,12 @@ module Docs self.release = '5.6' self.base_url = 'https://doc.qt.io/qt-5.6/' end + + def get_latest_version(options, &block) + fetch_doc('https://doc.qt.io/qt-5/index.html', options) do |doc| + label = doc.at_css('.mainContent h1.title').content + block.call label.sub(/Qt /, '') + end + end end end diff --git a/lib/docs/scrapers/ramda.rb b/lib/docs/scrapers/ramda.rb index 09ff46c8..d3751e38 100644 --- a/lib/docs/scrapers/ramda.rb +++ b/lib/docs/scrapers/ramda.rb @@ -15,6 +15,12 @@ module Docs © 2013–2016 Scott Sauyet and Michael Hurley
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://ramdajs.com/docs/', options) do |doc| + block.call doc.at_css('.navbar-brand > .version').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/react.rb b/lib/docs/scrapers/react.rb index 704f3189..54038244 100644 --- a/lib/docs/scrapers/react.rb +++ b/lib/docs/scrapers/react.rb @@ -30,5 +30,11 @@ module Docs © 2013–present Facebook Inc.
Licensed under the Creative Commons Attribution 4.0 International Public License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://reactjs.org/docs/getting-started.html', options) do |doc| + block.call doc.at_css('a[href="/versions"]').content.strip[1..-1] + end + end end end diff --git a/lib/docs/scrapers/react_native.rb b/lib/docs/scrapers/react_native.rb index d1f94001..ac5bd250 100644 --- a/lib/docs/scrapers/react_native.rb +++ b/lib/docs/scrapers/react_native.rb @@ -30,5 +30,11 @@ module Docs © 2015–2018 Facebook Inc.
Licensed under the Creative Commons Attribution 4.0 International Public License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', options) do |doc| + block.call doc.at_css('header > a > h3').content + end + end end end diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb index 201dbb47..89a67598 100644 --- a/lib/docs/scrapers/redis.rb +++ b/lib/docs/scrapers/redis.rb @@ -19,5 +19,12 @@ module Docs © 2009–2018 Salvatore Sanfilippo
Licensed under the Creative Commons Attribution-ShareAlike License 4.0. HTML + + def get_latest_version(options, &block) + fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', options) do |body| + body = body.lines[1..-1].join + block.call body.scan(/Redis ([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/redux.rb b/lib/docs/scrapers/redux.rb index cd7369ba..09738bb3 100644 --- a/lib/docs/scrapers/redux.rb +++ b/lib/docs/scrapers/redux.rb @@ -20,5 +20,9 @@ module Docs stub '' do request_one('http://redux.js.org/index.html').body end + + def get_latest_version(options, &block) + get_npm_version('redux', options, &block) + end end end diff --git a/lib/docs/scrapers/relay.rb b/lib/docs/scrapers/relay.rb index a020e9d7..90423ac6 100644 --- a/lib/docs/scrapers/relay.rb +++ b/lib/docs/scrapers/relay.rb @@ -18,5 +18,11 @@ module Docs © 2013–present Facebook Inc.
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + fetch_doc('http://facebook.github.io/relay/en/', options) do |doc| + block.call doc.at_css('header > a > h3').content[1..-1] + end + end end end diff --git a/lib/docs/scrapers/requirejs.rb b/lib/docs/scrapers/requirejs.rb index 200f1f2d..cf417a3e 100644 --- a/lib/docs/scrapers/requirejs.rb +++ b/lib/docs/scrapers/requirejs.rb @@ -30,5 +30,9 @@ module Docs © jQuery Foundation and other contributors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('requirejs', options, &block) + end end end diff --git a/lib/docs/scrapers/rethinkdb.rb b/lib/docs/scrapers/rethinkdb.rb index 023bbe77..27d913e0 100644 --- a/lib/docs/scrapers/rethinkdb.rb +++ b/lib/docs/scrapers/rethinkdb.rb @@ -58,6 +58,12 @@ module Docs CODE end + def get_latest_version(options, &block) + get_latest_github_release('rethinkdb', 'rethinkdb', options) do |release| + block.call release['tag_name'][1..-1] + end + end + private def process_response?(response) diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb index 1677e138..7c8d1519 100644 --- a/lib/docs/scrapers/rust.rb +++ b/lib/docs/scrapers/rust.rb @@ -39,6 +39,13 @@ module Docs Licensed under the Apache License, Version 2.0 or the MIT license, at your option. HTML + def get_latest_version(options, &block) + fetch_doc('https://www.rust-lang.org/', options) do |doc| + label = doc.at_css('.button-download + p > a').content + block.call label.sub(/Version /, '') + end + end + private REDIRECT_RGX = /http-equiv="refresh"/i diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb index 244a88e7..f493289b 100644 --- a/lib/docs/scrapers/sass.rb +++ b/lib/docs/scrapers/sass.rb @@ -23,5 +23,11 @@ module Docs © 2006–2016 Hampton Catlin, Nathan Weizenbaum, and Chris Eppstein
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_github_file_contents('sass', 'sass', 'VERSION', options) do |contents| + block.call contents.strip + end + end end end diff --git a/lib/docs/scrapers/scikit_image.rb b/lib/docs/scrapers/scikit_image.rb index 2a60aaec..3c454bcb 100644 --- a/lib/docs/scrapers/scikit_image.rb +++ b/lib/docs/scrapers/scikit_image.rb @@ -20,5 +20,11 @@ module Docs © 2011 the scikit-image team
Licensed under the BSD 3-clause License. HTML + + def get_latest_version(options, &block) + get_github_tags('scikit-image', 'scikit-image', options) do |tags| + block.call tags[0]['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/scikit_learn.rb b/lib/docs/scrapers/scikit_learn.rb index fe6a8665..0d7fc90d 100644 --- a/lib/docs/scrapers/scikit_learn.rb +++ b/lib/docs/scrapers/scikit_learn.rb @@ -25,5 +25,11 @@ module Docs Licensed under the 3-clause BSD License. HTML + def get_latest_version(options, &block) + fetch_doc('https://scikit-learn.org/stable/documentation.html', options) do |doc| + label = doc.at_css('.body h1').content + block.call label.scan(/([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/sinon.rb b/lib/docs/scrapers/sinon.rb index 9e81cba9..a1bd06cc 100644 --- a/lib/docs/scrapers/sinon.rb +++ b/lib/docs/scrapers/sinon.rb @@ -52,5 +52,11 @@ module Docs self.release = '1.17.7' self.base_url = "https://sinonjs.org/releases/v#{release}/" end + + def get_latest_version(options, &block) + fetch('https://sinonjs.org/', options) do |body| + block.call body.scan(/\/releases\/v([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/socketio.rb b/lib/docs/scrapers/socketio.rb index 427098c4..8518ff53 100644 --- a/lib/docs/scrapers/socketio.rb +++ b/lib/docs/scrapers/socketio.rb @@ -20,5 +20,9 @@ module Docs © 2014–2015 Automattic
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + get_npm_version('socket.io', options, &block) + end end end diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb index 790acf83..27720111 100644 --- a/lib/docs/scrapers/sqlite.rb +++ b/lib/docs/scrapers/sqlite.rb @@ -41,6 +41,12 @@ module Docs options[:attribution] = 'SQLite is in the Public Domain.' + def get_latest_version(options, &block) + fetch_doc('https://sqlite.org/chronology.html', options) do |doc| + block.call doc.at_css('#chrontab > tbody > tr > td:last-child > a').content + end + end + private def parse(response) diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb index 255bde6c..0c9bfd06 100644 --- a/lib/docs/scrapers/statsmodels.rb +++ b/lib/docs/scrapers/statsmodels.rb @@ -21,5 +21,10 @@ module Docs Licensed under the 3-clause BSD License. HTML + def get_latest_version(options, &block) + fetch_doc('http://www.statsmodels.org/stable/', options) do |doc| + block.call doc.at_css('.sphinxsidebarwrapper h3 + p > b').content + end + end end end diff --git a/lib/docs/scrapers/symfony.rb b/lib/docs/scrapers/symfony.rb index f5e4285a..acd105da 100644 --- a/lib/docs/scrapers/symfony.rb +++ b/lib/docs/scrapers/symfony.rb @@ -70,5 +70,11 @@ module Docs self.release = '2.7.35' self.base_url = "https://api.symfony.com/#{version}/" end + + def get_latest_version(options, &block) + get_latest_github_release('symfony', 'symfony', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/tcl_tk.rb b/lib/docs/scrapers/tcl_tk.rb index 1e4b2567..3dcc5765 100644 --- a/lib/docs/scrapers/tcl_tk.rb +++ b/lib/docs/scrapers/tcl_tk.rb @@ -25,5 +25,12 @@ module Docs options[:attribution] = <<-HTML Licensed under Tcl/Tk terms HTML + + def get_latest_version(options, &block) + fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', options) do |doc| + label = doc.at_css('h2').content + block.call label.scan(/Tk([0-9.]+)/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/tensorflow.rb b/lib/docs/scrapers/tensorflow.rb index 0296299f..dfd89875 100644 --- a/lib/docs/scrapers/tensorflow.rb +++ b/lib/docs/scrapers/tensorflow.rb @@ -56,6 +56,12 @@ module Docs /\Aextend/] end + def get_latest_version(options, &block) + get_latest_github_release('tensorflow', 'tensorflow', options) do |release| + block.call release['tag_name'][1..-1] + end + end + private def parse(response) diff --git a/lib/docs/scrapers/terraform.rb b/lib/docs/scrapers/terraform.rb index ab12775d..73d47e86 100644 --- a/lib/docs/scrapers/terraform.rb +++ b/lib/docs/scrapers/terraform.rb @@ -18,5 +18,11 @@ module Docs © 2018 HashiCorp
Licensed under the MPL 2.0 License. HTML + + def get_latest_version(options, &block) + get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', options) do |contents| + block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/twig.rb b/lib/docs/scrapers/twig.rb index 4781dcb2..58c7e2ef 100755 --- a/lib/docs/scrapers/twig.rb +++ b/lib/docs/scrapers/twig.rb @@ -28,5 +28,11 @@ module Docs self.release = '1.34.3' self.base_url = 'https://twig.symfony.com/doc/1.x/' end + + def get_latest_version(options, &block) + get_github_tags('twigphp', 'Twig', options) do |tags| + block.call tags[0]['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb index 1117314a..9e62d6a9 100644 --- a/lib/docs/scrapers/typescript.rb +++ b/lib/docs/scrapers/typescript.rb @@ -24,6 +24,12 @@ module Docs © Microsoft and other contributors
Licensed under the Apache License, Version 2.0. HTML + + def get_latest_version(options, &block) + get_latest_github_release('Microsoft', 'TypeScript', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/underscore.rb b/lib/docs/scrapers/underscore.rb index 004a09e9..127dd5d3 100644 --- a/lib/docs/scrapers/underscore.rb +++ b/lib/docs/scrapers/underscore.rb @@ -20,5 +20,11 @@ module Docs © 2009–2018 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
Licensed under the MIT License. HTML + + def get_latest_version(options, &block) + fetch_doc('https://underscorejs.org/', options) do |doc| + block.call doc.at_css('.version').content[1...-1] + end + end end end diff --git a/lib/docs/scrapers/vagrant.rb b/lib/docs/scrapers/vagrant.rb index 955746aa..0e89dad1 100644 --- a/lib/docs/scrapers/vagrant.rb +++ b/lib/docs/scrapers/vagrant.rb @@ -18,5 +18,11 @@ module Docs © 2010–2018 Mitchell Hashimoto
Licensed under the MPL 2.0 License. HTML + + def get_latest_version(options, &block) + get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', options) do |contents| + block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] + end + end end end diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb index f92991f2..cbb6b15e 100644 --- a/lib/docs/scrapers/vue.rb +++ b/lib/docs/scrapers/vue.rb @@ -32,5 +32,11 @@ module Docs self.root_path = '/guide/index.html' self.initial_paths = %w(/api/index.html) end + + def get_latest_version(options, &block) + get_latest_github_release('vuejs', 'vue', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/vulkan.rb b/lib/docs/scrapers/vulkan.rb index e1fa4b4b..111e405f 100644 --- a/lib/docs/scrapers/vulkan.rb +++ b/lib/docs/scrapers/vulkan.rb @@ -20,5 +20,11 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International License.
Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc. HTML + + def get_latest_version(options, &block) + get_github_tags('KhronosGroup', 'Vulkan-Docs', options) do |tags| + block.call tags[0]['name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/webpack.rb b/lib/docs/scrapers/webpack.rb index 255c1bb4..9d7303ed 100644 --- a/lib/docs/scrapers/webpack.rb +++ b/lib/docs/scrapers/webpack.rb @@ -68,5 +68,9 @@ module Docs Licensed under the MIT License. HTML end + + def get_latest_version(options, &block) + get_npm_version('webpack', options, &block) + end end end diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb index 825a749f..26f7c0cc 100644 --- a/lib/docs/scrapers/yarn.rb +++ b/lib/docs/scrapers/yarn.rb @@ -20,5 +20,11 @@ module Docs © 2016–present Yarn Contributors
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + get_latest_github_release('yarnpkg', 'yarn', options) do |release| + block.call release['tag_name'][1..-1] + end + end end end diff --git a/lib/docs/scrapers/yii.rb b/lib/docs/scrapers/yii.rb index d99d3ac7..0d087abb 100755 --- a/lib/docs/scrapers/yii.rb +++ b/lib/docs/scrapers/yii.rb @@ -34,5 +34,11 @@ module Docs options[:container] = '.grid_9' end + + def get_latest_version(options, &block) + get_latest_github_release('yiisoft', 'yii2', options) do |release| + block.call release['tag_name'] + end + end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index f370544c..6eff052e 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -58,7 +58,7 @@ class UpdatesCLI < Thor private - def check_doc(doc, options) + def check_doc(doc, opts) # Newer scraper versions always come before older scraper versions # Therefore, the first item's release value is the latest scraper version # @@ -71,7 +71,7 @@ class UpdatesCLI < Thor logger.debug("Checking #{doc.name}") - instance.get_latest_version(options) do |latest_version| + instance.get_latest_version(opts) do |latest_version| return { name: doc.name, scraper_version: scraper_version, From cd43632a2c44a1ac49d696a5267227b9b76503e1 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 01:10:31 +0100 Subject: [PATCH 08/17] Check for updates based on modified date for scrapers without release --- docs/Scraper-Reference.md | 2 +- lib/docs/core/doc.rb | 104 +++++++++++++++++++++++++++- lib/docs/core/scraper.rb | 85 ----------------------- lib/docs/scrapers/c.rb | 8 +++ lib/docs/scrapers/chef.rb | 5 +- lib/docs/scrapers/cpp.rb | 9 +++ lib/docs/scrapers/haskell.rb | 9 +-- lib/docs/scrapers/http.rb | 2 + lib/docs/scrapers/markdown.rb | 4 ++ lib/docs/scrapers/mdn/css.rb | 2 + lib/docs/scrapers/mdn/dom.rb | 2 + lib/docs/scrapers/mdn/dom_events.rb | 2 + lib/docs/scrapers/mdn/html.rb | 2 + lib/docs/scrapers/mdn/javascript.rb | 2 + lib/docs/scrapers/mdn/mdn.rb | 6 ++ lib/docs/scrapers/mdn/svg.rb | 2 + lib/docs/scrapers/mdn/xslt_xpath.rb | 2 + lib/docs/scrapers/openjdk.rb | 2 +- lib/docs/scrapers/support_tables.rb | 7 ++ lib/tasks/updates.thor | 44 ++++++------ 20 files changed, 186 insertions(+), 115 deletions(-) diff --git a/docs/Scraper-Reference.md b/docs/Scraper-Reference.md index 60d377d8..a6736a3e 100644 --- a/docs/Scraper-Reference.md +++ b/docs/Scraper-Reference.md @@ -187,7 +187,7 @@ More information about how filters work is available on the [Filter Reference](. ## Keeping scrapers up-to-date -In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden by all scrapers that define the `self.release` attribute. This method should return the latest version of the documentation that is being scraped. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. +In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. To make life easier, there are a few utility methods that you can use in `get_latest_version`: * `fetch(url, options, &block)` diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index cb1cd209..062ac9e7 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -152,7 +152,6 @@ module Docs end end - def initialize raise NotImplementedError, "#{self.class} is an abstract class and cannot be instantiated." if self.class.abstract end @@ -164,5 +163,108 @@ module Docs def build_pages(&block) raise NotImplementedError end + + def get_scraper_version(opts, &block) + if self.class.method_defined?(:options) and !options[:release].nil? + block.call options[:release] + else + # If options[:release] does not exist, we return the Epoch timestamp of when the doc was last modified in DevDocs production + fetch_json('https://devdocs.io/docs.json', opts) do |json| + items = json.select {|item| item['name'] == self.class.name} + items = items.map {|item| item['mtime']} + block.call items.max + end + end + end + + # Should return the latest version of this documentation + # If options[:release] is defined, it should be in the same format + # If options[:release] is not defined, it should return the Epoch timestamp of when the documentation was last updated + # If the docs will never change, simply return '1.0.0' + def get_latest_version(options, &block) + raise NotImplementedError + end + + # Returns whether or not this scraper is outdated. + # + # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions. + # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases. + # + # Scrapers of documentations that do not use this versioning approach should override this method. + # + # Examples of the default implementation: + # 1 -> 2 = outdated + # 1.1 -> 1.2 = outdated + # 1.1.1 -> 1.1.2 = not outdated + def is_outdated(scraper_version, latest_version) + scraper_parts = scraper_version.to_s.split(/\./).map(&:to_i) + latest_parts = latest_version.to_s.split(/\./).map(&:to_i) + + # Only check the first two parts, the third part is for patch updates + [0, 1].each do |i| + break if i >= scraper_parts.length or i >= latest_parts.length + return true if latest_parts[i] > scraper_parts[i] + return false if latest_parts[i] < scraper_parts[i] + end + + false + end + + private + + # + # Utility methods for get_latest_version + # + + def fetch(url, options, &block) + headers = {} + + if options.key?(:github_token) and url.start_with?('https://api.github.com/') + headers['Authorization'] = "token #{options[:github_token]}" + end + + options[:logger].debug("Fetching #{url}") + + Request.run(url, { headers: headers }) do |response| + if response.success? + block.call response.body + else + options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") + block.call nil + end + end + end + + def fetch_doc(url, options, &block) + fetch(url, options) do |body| + block.call Nokogiri::HTML.parse(body, nil, 'UTF-8') + end + end + + def fetch_json(url, options, &block) + fetch(url, options) do |body| + block.call JSON.parse(body) + end + end + + def get_npm_version(package, options, &block) + fetch_json("https://registry.npmjs.com/#{package}", options) do |json| + block.call json['dist-tags']['latest'] + end + end + + def get_latest_github_release(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) + end + + def get_github_tags(owner, repo, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) + end + + def get_github_file_contents(owner, repo, path, options, &block) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| + block.call(Base64.decode64(json['content'])) + end + end end end diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index b124c6db..083b0015 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -132,35 +132,6 @@ module Docs end end - def get_latest_version(options, &block) - raise NotImplementedError - end - - # Returns whether or not this scraper is outdated. - # - # The default implementation assumes the documentation uses a semver(-like) approach when it comes to versions. - # Patch updates are ignored because there are usually little to no documentation changes in bug-fix-only releases. - # - # Scrapers of documentations that do not use this versioning approach should override this method. - # - # Examples of the default implementation: - # 1 -> 2 = outdated - # 1.1 -> 1.2 = outdated - # 1.1.1 -> 1.1.2 = not outdated - def is_outdated(scraper_version, latest_version) - scraper_parts = scraper_version.split(/\./).map(&:to_i) - latest_parts = latest_version.split(/\./).map(&:to_i) - - # Only check the first two parts, the third part is for patch updates - [0, 1].each do |i| - break if i >= scraper_parts.length or i >= latest_parts.length - return true if latest_parts[i] > scraper_parts[i] - return false if latest_parts[i] < scraper_parts[i] - end - - false - end - private def request_one(url) @@ -231,62 +202,6 @@ module Docs {} end - # - # Utility methods for get_latest_version - # - - def fetch(url, options, &block) - headers = {} - - if options.key?(:github_token) and url.start_with?('https://api.github.com/') - headers['Authorization'] = "token #{options[:github_token]}" - end - - options[:logger].debug("Fetching #{url}") - - Request.run(url, { headers: headers }) do |response| - if response.success? - block.call response.body - else - options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") - block.call nil - end - end - end - - def fetch_doc(url, options, &block) - fetch(url, options) do |body| - block.call Nokogiri::HTML.parse body, nil, 'UTF-8' - end - end - - def fetch_json(url, options, &block) - fetch(url, options) do |body| - json = JSON.parse(body) - block.call json - end - end - - def get_npm_version(package, options, &block) - fetch_json("https://registry.npmjs.com/#{package}", options) do |json| - block.call json['dist-tags']['latest'] - end - end - - def get_latest_github_release(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) - end - - def get_github_tags(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) - end - - def get_github_file_contents(owner, repo, path, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| - block.call(Base64.decode64(json['content'])) - end - end - module FixInternalUrlsBehavior def self.included(base) base.extend ClassMethods diff --git a/lib/docs/scrapers/c.rb b/lib/docs/scrapers/c.rb index f9289617..0ab0ac39 100644 --- a/lib/docs/scrapers/c.rb +++ b/lib/docs/scrapers/c.rb @@ -26,6 +26,14 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML + def get_latest_version(options, &block) + fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i + end + end + private def file_path_for(*) diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb index 337d1202..f9a248bd 100644 --- a/lib/docs/scrapers/chef.rb +++ b/lib/docs/scrapers/chef.rb @@ -49,9 +49,8 @@ module Docs end def get_latest_version(options, &block) - fetch_doc('https://docs-archive.chef.io/', options) do |doc| - cell = doc.at_css('.main-archives > tr:nth-child(2) > td:nth-child(2)') - block.call cell.content.sub(/Chef Client /, '') + fetch_doc('https://downloads.chef.io/chef', options) do |doc| + block.call doc.at_css('h1.product-heading > span').content.strip end end end diff --git a/lib/docs/scrapers/cpp.rb b/lib/docs/scrapers/cpp.rb index 374f6883..d26eae6a 100644 --- a/lib/docs/scrapers/cpp.rb +++ b/lib/docs/scrapers/cpp.rb @@ -34,6 +34,15 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML + # Same as get_latest_version in lib/docs/scrapers/c.rb + def get_latest_version(options, &block) + fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i + end + end + private def file_path_for(*) diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index 383e1990..fc848a7a 100755 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -10,7 +10,7 @@ module Docs html_filters.push 'haskell/entries', 'haskell/clean_html' - options[:container] = ->(filter) { filter.subpath.start_with?('users_guide') ? '.body' : '#content' } + options[:container] = ->(filter) {filter.subpath.start_with?('users_guide') ? '.body' : '#content'} options[:only_patterns] = [/\Alibraries\//, /\Ausers_guide\//] options[:skip_patterns] = [ @@ -70,9 +70,10 @@ module Docs end def get_latest_version(options, &block) - fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/users_guide/', options) do |doc| - label = doc.at_css('.related > ul > li:last-child').content - block.call label.scan(/([0-9.]+)/)[0][0] + fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', options) do |doc| + links = doc.css('a').to_a + versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)} + block.call versions.find {|version| !version.empty?}[0][0] end end end diff --git a/lib/docs/scrapers/http.rb b/lib/docs/scrapers/http.rb index 60f15f75..90e6f5c1 100644 --- a/lib/docs/scrapers/http.rb +++ b/lib/docs/scrapers/http.rb @@ -7,6 +7,8 @@ module Docs html_filters.push 'http/clean_html', 'http/entries', 'title' + options[:mdn_tag] = 'HTTP' + options[:root_title] = 'HTTP' options[:title] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title } options[:container] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? '.content' : nil } diff --git a/lib/docs/scrapers/markdown.rb b/lib/docs/scrapers/markdown.rb index 87e9c957..3400e270 100644 --- a/lib/docs/scrapers/markdown.rb +++ b/lib/docs/scrapers/markdown.rb @@ -13,5 +13,9 @@ module Docs © 2004 John Gruber
Licensed under the BSD License. HTML + + def get_latest_version(options, &block) + block.call '1.0.0' + end end end diff --git a/lib/docs/scrapers/mdn/css.rb b/lib/docs/scrapers/mdn/css.rb index 4c44f1f1..abb69b3a 100644 --- a/lib/docs/scrapers/mdn/css.rb +++ b/lib/docs/scrapers/mdn/css.rb @@ -6,6 +6,8 @@ module Docs html_filters.push 'css/clean_html', 'css/entries', 'title' + options[:mdn_tag] = 'CSS' + options[:root_title] = 'CSS' options[:skip] = %w(/CSS3 /Media/Visual /paged_media /Media/TV /Media/Tactile) diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb index bbf95b20..a2202929 100644 --- a/lib/docs/scrapers/mdn/dom.rb +++ b/lib/docs/scrapers/mdn/dom.rb @@ -8,6 +8,8 @@ module Docs html_filters.push 'dom/clean_html', 'dom/entries', 'title' + options[:mdn_tag] = 'XSLT_Reference' + options[:root_title] = 'DOM' options[:skip] = %w( diff --git a/lib/docs/scrapers/mdn/dom_events.rb b/lib/docs/scrapers/mdn/dom_events.rb index fcbdc08f..258fbcd4 100644 --- a/lib/docs/scrapers/mdn/dom_events.rb +++ b/lib/docs/scrapers/mdn/dom_events.rb @@ -9,6 +9,8 @@ module Docs html_filters.insert_after 'clean_html', 'dom_events/clean_html' html_filters.push 'dom_events/entries', 'title' + options[:mdn_tag] = 'events' + options[:root_title] = 'DOM Events' options[:skip] = %w(/MozOrientation) diff --git a/lib/docs/scrapers/mdn/html.rb b/lib/docs/scrapers/mdn/html.rb index 4b28cefd..f38432f1 100644 --- a/lib/docs/scrapers/mdn/html.rb +++ b/lib/docs/scrapers/mdn/html.rb @@ -7,6 +7,8 @@ module Docs html_filters.push 'html/clean_html', 'html/entries', 'title' + options[:mdn_tag] = 'HTML' + options[:root_title] = 'HTML' options[:title] = ->(filter) do diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb index 935df61c..cea55fc8 100644 --- a/lib/docs/scrapers/mdn/javascript.rb +++ b/lib/docs/scrapers/mdn/javascript.rb @@ -8,6 +8,8 @@ module Docs html_filters.push 'javascript/clean_html', 'javascript/entries', 'title' + options[:mdn_tag] = 'JavaScript' + options[:root_title] = 'JavaScript' # Don't want diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb index 2ebd38aa..ccb27af9 100644 --- a/lib/docs/scrapers/mdn/mdn.rb +++ b/lib/docs/scrapers/mdn/mdn.rb @@ -21,6 +21,12 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later. HTML + def get_latest_version(opts, &block) + fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts) do |json| + block.call DateTime.parse(json[0]['pubdate']).to_time.to_i + end + end + private def process_response?(response) diff --git a/lib/docs/scrapers/mdn/svg.rb b/lib/docs/scrapers/mdn/svg.rb index db9de7a1..66baf60d 100644 --- a/lib/docs/scrapers/mdn/svg.rb +++ b/lib/docs/scrapers/mdn/svg.rb @@ -8,6 +8,8 @@ module Docs html_filters.push 'svg/clean_html', 'svg/entries', 'title' + options[:mdn_tag] = 'XSLT_Reference' + options[:root_title] = 'SVG' options[:title] = ->(filter) do diff --git a/lib/docs/scrapers/mdn/xslt_xpath.rb b/lib/docs/scrapers/mdn/xslt_xpath.rb index 5d812dd4..9bf01c01 100644 --- a/lib/docs/scrapers/mdn/xslt_xpath.rb +++ b/lib/docs/scrapers/mdn/xslt_xpath.rb @@ -8,6 +8,8 @@ module Docs html_filters.push 'xslt_xpath/clean_html', 'xslt_xpath/entries', 'title' + options[:mdn_tag] = 'XSLT_Reference' + options[:root_title] = 'XSLT' options[:only_patterns] = [/\A\/XSLT/, /\A\/XPath/] diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb index e833c7c4..a56a2928 100644 --- a/lib/docs/scrapers/openjdk.rb +++ b/lib/docs/scrapers/openjdk.rb @@ -105,7 +105,7 @@ module Docs end end - block.call latest_version.to_s + block.call latest_version end end end diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb index 9a550c1e..90cd7f4f 100644 --- a/lib/docs/scrapers/support_tables.rb +++ b/lib/docs/scrapers/support_tables.rb @@ -178,5 +178,12 @@ module Docs

HTML + + def get_latest_version(options, &block) + fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', options) do |body| + timestamp = body.scan(/([^<]+)<\/updated>/)[0][0] + block.call DateTime.parse(timestamp).to_time.to_i + end + end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index 6eff052e..e684cdbc 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -1,7 +1,7 @@ class UpdatesCLI < Thor # The GitHub user that is allowed to upload reports # TODO: Update this before creating a PR - UPLOAD_USER = 'jmerle' + UPLOAD_USER = 'devdocs-bot' # The repository to create an issue in when uploading the results # TODO: Update this before creating a PR @@ -59,25 +59,19 @@ class UpdatesCLI < Thor private def check_doc(doc, opts) - # Newer scraper versions always come before older scraper versions - # Therefore, the first item's release value is the latest scraper version - # - # For example, a scraper could scrape 3 versions: 10, 11 and 12 - # doc.versions.first would be the scraper for version 12 - instance = doc.versions.first.new - - scraper_version = instance.class.method_defined?(:options) ? instance.options[:release] : nil - return error_result(doc, '`options[:release]` does not exist') if scraper_version.nil? - logger.debug("Checking #{doc.name}") - instance.get_latest_version(opts) do |latest_version| - return { - name: doc.name, - scraper_version: scraper_version, - latest_version: latest_version, - is_outdated: instance.is_outdated(scraper_version, latest_version) - } + instance = doc.versions.first.new + + instance.get_scraper_version(opts) do |scraper_version| + instance.get_latest_version(opts) do |latest_version| + return { + name: doc.name, + scraper_version: format_version(scraper_version), + latest_version: format_version(latest_version), + is_outdated: instance.is_outdated(scraper_version, latest_version) + } + end end rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") @@ -87,6 +81,15 @@ class UpdatesCLI < Thor raise end + def format_version(version) + str = version.to_s + + # If the version is numeric and greater than or equal to 1e9 it's probably a timestamp + return str if str.match(/^(\d)+$/).nil? or str.to_i < 1e9 + + DateTime.strptime(str, '%s').strftime('%B %-d, %Y') + end + def error_result(doc, reason) { name: doc.name, @@ -199,14 +202,15 @@ class UpdatesCLI < Thor ] results_str = results.select {|result| !result.nil?}.join("\n\n") + travis_str = ENV['TRAVIS'].nil? ? '' : "\n\nThis issue was created by Travis CI build [##{ENV['TRAVIS_BUILD_NUMBER']}](#{ENV['TRAVIS_BUILD_WEB_URL']})." - title = "Documentation versions report for #{Date.today.strftime('%B')} 2019" + title = "Documentation versions report for #{Date.today.strftime('%B %Y')}" body = <<-MARKDOWN ## What is this? This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations. -Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created. +Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created.#{travis_str} ## Results From 41e0a138a6533ed39f88d242db051ce33287e505 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 03:02:24 +0100 Subject: [PATCH 09/17] Refactoring and cleaning up --- docs/Scraper-Reference.md | 32 +++--- lib/docs/core/doc.rb | 71 ++++++------ lib/docs/scrapers/angular.rb | 4 +- lib/docs/scrapers/angularjs.rb | 4 +- lib/docs/scrapers/ansible.rb | 7 +- lib/docs/scrapers/apache.rb | 7 +- lib/docs/scrapers/apache_pig.rb | 9 +- lib/docs/scrapers/async.rb | 8 +- lib/docs/scrapers/babel.rb | 7 +- lib/docs/scrapers/backbone.rb | 8 +- lib/docs/scrapers/bash.rb | 8 +- lib/docs/scrapers/bluebird.rb | 4 +- lib/docs/scrapers/bootstrap.rb | 7 +- lib/docs/scrapers/bottle.rb | 9 +- lib/docs/scrapers/bower.rb | 4 +- lib/docs/scrapers/c.rb | 11 +- lib/docs/scrapers/cakephp.rb | 7 +- lib/docs/scrapers/chai.rb | 4 +- lib/docs/scrapers/chef.rb | 7 +- lib/docs/scrapers/clojure.rb | 7 +- lib/docs/scrapers/cmake.rb | 9 +- lib/docs/scrapers/codeception.rb | 7 +- lib/docs/scrapers/codeceptjs.rb | 4 +- lib/docs/scrapers/codeigniter.rb | 9 +- lib/docs/scrapers/coffeescript.rb | 4 +- lib/docs/scrapers/cordova.rb | 15 +-- lib/docs/scrapers/cpp.rb | 11 +- lib/docs/scrapers/crystal.rb | 7 +- lib/docs/scrapers/d.rb | 7 +- lib/docs/scrapers/d3.rb | 4 +- lib/docs/scrapers/dart.rb | 9 +- lib/docs/scrapers/django.rb | 7 +- lib/docs/scrapers/docker.rb | 9 +- lib/docs/scrapers/dojo.rb | 7 +- lib/docs/scrapers/drupal.rb | 15 +-- lib/docs/scrapers/electron.rb | 7 +- lib/docs/scrapers/elixir.rb | 7 +- lib/docs/scrapers/ember.rb | 7 +- lib/docs/scrapers/erlang.rb | 7 +- lib/docs/scrapers/eslint.rb | 4 +- lib/docs/scrapers/express.rb | 4 +- lib/docs/scrapers/falcon.rb | 7 +- lib/docs/scrapers/fish.rb | 7 +- lib/docs/scrapers/flow.rb | 4 +- lib/docs/scrapers/git.rb | 7 +- lib/docs/scrapers/gnu/gcc.rb | 9 +- lib/docs/scrapers/gnu/gnu_fortran.rb | 9 +- lib/docs/scrapers/go.rb | 15 +-- lib/docs/scrapers/godot.rb | 7 +- lib/docs/scrapers/graphite.rb | 7 +- lib/docs/scrapers/grunt.rb | 4 +- lib/docs/scrapers/handlebars.rb | 4 +- lib/docs/scrapers/haskell.rb | 11 +- lib/docs/scrapers/haxe.rb | 9 +- lib/docs/scrapers/homebrew.rb | 6 +- lib/docs/scrapers/immutable.rb | 4 +- lib/docs/scrapers/influxdata.rb | 9 +- lib/docs/scrapers/jasmine.rb | 6 +- lib/docs/scrapers/jekyll.rb | 7 +- lib/docs/scrapers/jest.rb | 7 +- lib/docs/scrapers/jquery/jquery_core.rb | 4 +- lib/docs/scrapers/jquery/jquery_mobile.rb | 8 +- lib/docs/scrapers/jquery/jquery_ui.rb | 4 +- lib/docs/scrapers/jsdoc.rb | 6 +- lib/docs/scrapers/julia.rb | 6 +- lib/docs/scrapers/knockout.rb | 6 +- lib/docs/scrapers/koa.rb | 4 +- lib/docs/scrapers/kotlin.rb | 6 +- lib/docs/scrapers/laravel.rb | 6 +- lib/docs/scrapers/leaflet.rb | 9 +- lib/docs/scrapers/less.rb | 9 +- lib/docs/scrapers/liquid.rb | 7 +- lib/docs/scrapers/lodash.rb | 7 +- lib/docs/scrapers/love.rb | 7 +- lib/docs/scrapers/lua.rb | 7 +- lib/docs/scrapers/marionette.rb | 4 +- lib/docs/scrapers/markdown.rb | 4 +- lib/docs/scrapers/matplotlib.rb | 6 +- lib/docs/scrapers/mdn/mdn.rb | 7 +- lib/docs/scrapers/meteor.rb | 7 +- lib/docs/scrapers/mocha.rb | 4 +- lib/docs/scrapers/modernizr.rb | 4 +- lib/docs/scrapers/moment.rb | 7 +- lib/docs/scrapers/mongoose.rb | 9 +- lib/docs/scrapers/nginx.rb | 9 +- lib/docs/scrapers/nginx_lua_module.rb | 9 +- lib/docs/scrapers/nim.rb | 7 +- lib/docs/scrapers/node.rb | 7 +- lib/docs/scrapers/nokogiri2.rb | 6 +- lib/docs/scrapers/npm.rb | 6 +- lib/docs/scrapers/numpy.rb | 6 +- lib/docs/scrapers/openjdk.rb | 17 ++- lib/docs/scrapers/opentsdb.rb | 6 +- lib/docs/scrapers/padrino.rb | 6 +- lib/docs/scrapers/pandas.rb | 9 +- lib/docs/scrapers/perl.rb | 7 +- lib/docs/scrapers/phalcon.rb | 7 +- lib/docs/scrapers/phaser.rb | 6 +- lib/docs/scrapers/phoenix.rb | 7 +- lib/docs/scrapers/php.rb | 9 +- lib/docs/scrapers/phpunit.rb | 9 +- lib/docs/scrapers/postgresql.rb | 9 +- lib/docs/scrapers/pug.rb | 4 +- lib/docs/scrapers/puppeteer.rb | 7 +- lib/docs/scrapers/pygame.rb | 6 +- lib/docs/scrapers/python.rb | 7 +- lib/docs/scrapers/q.rb | 4 +- lib/docs/scrapers/qt.rb | 8 +- lib/docs/scrapers/ramda.rb | 7 +- lib/docs/scrapers/rdoc/minitest.rb | 7 +- lib/docs/scrapers/rdoc/rails.rb | 6 +- lib/docs/scrapers/rdoc/ruby.rb | 15 ++- lib/docs/scrapers/react.rb | 7 +- lib/docs/scrapers/react_native.rb | 7 +- lib/docs/scrapers/redis.rb | 9 +- lib/docs/scrapers/redux.rb | 4 +- lib/docs/scrapers/relay.rb | 7 +- lib/docs/scrapers/requirejs.rb | 4 +- lib/docs/scrapers/rethinkdb.rb | 6 +- lib/docs/scrapers/rust.rb | 9 +- lib/docs/scrapers/sass.rb | 6 +- lib/docs/scrapers/scikit_image.rb | 7 +- lib/docs/scrapers/scikit_learn.rb | 8 +- lib/docs/scrapers/sinon.rb | 7 +- lib/docs/scrapers/socketio.rb | 4 +- lib/docs/scrapers/sqlite.rb | 7 +- lib/docs/scrapers/statsmodels.rb | 7 +- lib/docs/scrapers/support_tables.rb | 9 +- lib/docs/scrapers/symfony.rb | 6 +- lib/docs/scrapers/tcl_tk.rb | 8 +- lib/docs/scrapers/tensorflow.rb | 6 +- lib/docs/scrapers/terraform.rb | 7 +- lib/docs/scrapers/twig.rb | 7 +- lib/docs/scrapers/typescript.rb | 6 +- lib/docs/scrapers/underscore.rb | 7 +- lib/docs/scrapers/vagrant.rb | 7 +- lib/docs/scrapers/vue.rb | 6 +- lib/docs/scrapers/vulkan.rb | 7 +- lib/docs/scrapers/webpack.rb | 4 +- lib/docs/scrapers/yarn.rb | 6 +- lib/docs/scrapers/yii.rb | 6 +- lib/tasks/updates.thor | 130 +++++++++++----------- 142 files changed, 530 insertions(+), 676 deletions(-) diff --git a/docs/Scraper-Reference.md b/docs/Scraper-Reference.md index a6736a3e..de7d3f15 100644 --- a/docs/Scraper-Reference.md +++ b/docs/Scraper-Reference.md @@ -187,39 +187,41 @@ More information about how filters work is available on the [Filter Reference](. ## Keeping scrapers up-to-date -In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. +In order to keep scrapers up-to-date the `get_latest_version(opts)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. To make life easier, there are a few utility methods that you can use in `get_latest_version`: -* `fetch(url, options, &block)` +* `fetch(url, opts)` - Makes a GET request to the url and calls `&block` with the body. + Makes a GET request to the url and returns the response body. Example: [lib/docs/scrapers/bash.rb](../lib/docs/scrapers/bash.rb) -* `fetch_doc(url, options, &block)` +* `fetch_doc(url, opts)` - Makes a GET request to the url and calls `&block` with the HTML body converted to a Nokogiri document. + Makes a GET request to the url and returns the HTML body converted to a Nokogiri document. Example: [lib/docs/scrapers/git.rb](../lib/docs/scrapers/git.rb) -* `fetch_json(url, options, &block)` +* `fetch_json(url, opts)` - Makes a GET request to the url and calls `&block` with the JSON body converted to a dictionary. -* `get_npm_version(package, options, &block)` + Makes a GET request to the url and returns the JSON body converted to a dictionary. - Calls `&block` with the latest version of the given npm package. + Example: [lib/docs/scrapers/mdn/mdn.rb](../lib/docs/scrapers/mdn/mdn.rb) +* `get_npm_version(package, opts)` + + Returns the latest version of the given npm package. Example: [lib/docs/scrapers/bower.rb](../lib/docs/scrapers/bower.rb) -* `get_latest_github_release(owner, repo, options, &block)` +* `get_latest_github_release(owner, repo, opts)` - Calls `&block` with the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). + Returns the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). Example: [lib/docs/scrapers/jsdoc.rb](../lib/docs/scrapers/jsdoc.rb) -* `get_github_tags(owner, repo, options, &block)` +* `get_github_tags(owner, repo, opts)` - Calls `&block` with the list of tags on the given repository ([format](https://developer.github.com/v3/repos/#list-tags)). + Returns the list of tags on the given repository ([format](https://developer.github.com/v3/repos/#list-tags)). Example: [lib/docs/scrapers/liquid.rb](../lib/docs/scrapers/liquid.rb) -* `get_github_file_contents(owner, repo, path, options, &block)` +* `get_github_file_contents(owner, repo, path, opts)` - Calls `&block` with the contents of the requested file in the default branch of the given repository. + Returns the contents of the requested file in the default branch of the given repository. Example: [lib/docs/scrapers/minitest.rb](../lib/docs/scrapers/minitest.rb) diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index 062ac9e7..0b913ef6 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -164,16 +164,15 @@ module Docs raise NotImplementedError end - def get_scraper_version(opts, &block) + def get_scraper_version(opts) if self.class.method_defined?(:options) and !options[:release].nil? - block.call options[:release] + options[:release] else # If options[:release] does not exist, we return the Epoch timestamp of when the doc was last modified in DevDocs production - fetch_json('https://devdocs.io/docs.json', opts) do |json| - items = json.select {|item| item['name'] == self.class.name} - items = items.map {|item| item['mtime']} - block.call items.max - end + json = fetch_json('https://devdocs.io/docs.json', opts) + items = json.select {|item| item['name'] == self.class.name} + items = items.map {|item| item['mtime']} + items.max end end @@ -181,7 +180,7 @@ module Docs # If options[:release] is defined, it should be in the same format # If options[:release] is not defined, it should return the Epoch timestamp of when the documentation was last updated # If the docs will never change, simply return '1.0.0' - def get_latest_version(options, &block) + def get_latest_version(opts) raise NotImplementedError end @@ -216,55 +215,49 @@ module Docs # Utility methods for get_latest_version # - def fetch(url, options, &block) + def fetch(url, opts) headers = {} - if options.key?(:github_token) and url.start_with?('https://api.github.com/') - headers['Authorization'] = "token #{options[:github_token]}" + if opts.key?(:github_token) and url.start_with?('https://api.github.com/') + headers['Authorization'] = "token #{opts[:github_token]}" end - options[:logger].debug("Fetching #{url}") + opts[:logger].debug("Fetching #{url}") + response = Request.run(url, { headers: headers }) - Request.run(url, { headers: headers }) do |response| - if response.success? - block.call response.body - else - options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") - block.call nil - end + if response.success? + response.body + else + opts[:logger].error("Couldn't fetch #{url} (response code #{response.code})") + nil end end - def fetch_doc(url, options, &block) - fetch(url, options) do |body| - block.call Nokogiri::HTML.parse(body, nil, 'UTF-8') - end + def fetch_doc(url, opts) + body = fetch(url, opts) + Nokogiri::HTML.parse(body, nil, 'UTF-8') end - def fetch_json(url, options, &block) - fetch(url, options) do |body| - block.call JSON.parse(body) - end + def fetch_json(url, opts) + JSON.parse fetch(url, opts) end - def get_npm_version(package, options, &block) - fetch_json("https://registry.npmjs.com/#{package}", options) do |json| - block.call json['dist-tags']['latest'] - end + def get_npm_version(package, opts) + json = fetch_json("https://registry.npmjs.com/#{package}", opts) + json['dist-tags']['latest'] end - def get_latest_github_release(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) + def get_latest_github_release(owner, repo, opts) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", opts) end - def get_github_tags(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) + def get_github_tags(owner, repo, opts) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", opts) end - def get_github_file_contents(owner, repo, path, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| - block.call(Base64.decode64(json['content'])) - end + def get_github_file_contents(owner, repo, path, opts) + json = fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", opts) + Base64.decode64(json['content']) end end end diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index 059b0e8e..3365ec67 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -155,8 +155,8 @@ module Docs end end - def get_latest_version(options, &block) - get_npm_version('@angular/core', options, &block) + def get_latest_version(opts) + get_npm_version('@angular/core', opts) end private diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb index b6e18325..9d663e35 100644 --- a/lib/docs/scrapers/angularjs.rb +++ b/lib/docs/scrapers/angularjs.rb @@ -70,8 +70,8 @@ module Docs self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end - def get_latest_version(options, &block) - get_npm_version('angular', options, &block) + def get_latest_version(opts) + get_npm_version('angular', opts) end end end diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb index 293f74a7..b2363d4d 100644 --- a/lib/docs/scrapers/ansible.rb +++ b/lib/docs/scrapers/ansible.rb @@ -88,10 +88,9 @@ module Docs list_of_all_modules.html) end - def get_latest_version(options, &block) - fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc| - block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.ansible.com/ansible/latest/index.html', opts) + doc.at_css('.DocSiteProduct-CurrentVersion').content.strip end end end diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb index ba0fa340..1301b574 100644 --- a/lib/docs/scrapers/apache.rb +++ b/lib/docs/scrapers/apache.rb @@ -34,10 +34,9 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - fetch_doc('http://httpd.apache.org/docs/', options) do |doc| - block.call doc.at_css('#apcontents > ul a')['href'][0...-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://httpd.apache.org/docs/', opts) + doc.at_css('#apcontents > ul a')['href'][0...-1] end end end diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb index 5454140b..f35085e6 100644 --- a/lib/docs/scrapers/apache_pig.rb +++ b/lib/docs/scrapers/apache_pig.rb @@ -43,11 +43,10 @@ module Docs self.base_url = "https://pig.apache.org/docs/r#{release}/" end - def get_latest_version(options, &block) - fetch_doc('https://pig.apache.org/', options) do |doc| - item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') - block.call item.content.strip.sub(/Release /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://pig.apache.org/', opts) + item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') + item.content.strip.sub(/Release /, '') end end end diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 18e9bbbf..67498eed 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -18,11 +18,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://caolan.github.io/async/', options) do |doc| - version = doc.at_css('#version-dropdown > a').content.strip[1..-1] - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://caolan.github.io/async/', opts) + doc.at_css('#version-dropdown > a').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb index 675f86be..c8d716f1 100644 --- a/lib/docs/scrapers/babel.rb +++ b/lib/docs/scrapers/babel.rb @@ -23,10 +23,9 @@ module Docs '
' end - def get_latest_version(options, &block) - fetch_doc('https://babeljs.io/docs/en/', options) do |doc| - block.call doc.at_css('a[href="/versions"] > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://babeljs.io/docs/en/', opts) + doc.at_css('a[href="/versions"] > h3').content end end end diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb index ad6220e5..2b33505e 100644 --- a/lib/docs/scrapers/backbone.rb +++ b/lib/docs/scrapers/backbone.rb @@ -21,11 +21,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://backbonejs.org/', options) do |doc| - version = doc.at_css('.version').content - block.call version[1...-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://backbonejs.org/', opts) + doc.at_css('.version').content[1...-1] end end end diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb index 5556f5b9..92ef894a 100644 --- a/lib/docs/scrapers/bash.rb +++ b/lib/docs/scrapers/bash.rb @@ -18,11 +18,9 @@ module Docs Licensed under the GNU Free Documentation License. HTML - def get_latest_version(options, &block) - fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body| - version = body.scan(/, Version ([0-9.]+)/)[0][0] - block.call version[0...-1] - end + def get_latest_version(opts) + body = fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', opts) + body.scan(/, Version ([0-9.]+)/)[0][0][0...-1] end end end diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb index 8a960b87..8f38120a 100644 --- a/lib/docs/scrapers/bluebird.rb +++ b/lib/docs/scrapers/bluebird.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('bluebird', options, &block) + def get_latest_version(opts) + get_npm_version('bluebird', opts) end end end diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb index aa0b4cc3..8571462e 100644 --- a/lib/docs/scrapers/bootstrap.rb +++ b/lib/docs/scrapers/bootstrap.rb @@ -35,10 +35,9 @@ module Docs options[:only] = %w(getting-started/ css/ components/ javascript/) end - def get_latest_version(options, &block) - fetch_doc('https://getbootstrap.com/', options) do |doc| - block.call doc.at_css('#bd-versions').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://getbootstrap.com/', opts) + doc.at_css('#bd-versions').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/bottle.rb b/lib/docs/scrapers/bottle.rb index 6e4a19a8..d0397ec7 100644 --- a/lib/docs/scrapers/bottle.rb +++ b/lib/docs/scrapers/bottle.rb @@ -28,11 +28,10 @@ module Docs self.base_url = "https://bottlepy.org/docs/#{self.version}/" end - def get_latest_version(options, &block) - fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc| - label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') - block.call label.content.sub(/Bottle /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://bottlepy.org/docs/stable/', opts) + label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') + label.content.sub(/Bottle /, '') end end end diff --git a/lib/docs/scrapers/bower.rb b/lib/docs/scrapers/bower.rb index 1102ee75..aab2a1e9 100644 --- a/lib/docs/scrapers/bower.rb +++ b/lib/docs/scrapers/bower.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('bower', options, &block) + def get_latest_version(opts) + get_npm_version('bower', opts) end end end diff --git a/lib/docs/scrapers/c.rb b/lib/docs/scrapers/c.rb index 0ab0ac39..ec99f704 100644 --- a/lib/docs/scrapers/c.rb +++ b/lib/docs/scrapers/c.rb @@ -26,12 +26,11 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| - link = doc.at_css('a[title^="File:"]') - date = link.content.scan(/(\d+)\./)[0][0] - block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i - end + def get_latest_version(opts) + doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts) + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + DateTime.strptime(date, '%Y%m%d').to_time.to_i end private diff --git a/lib/docs/scrapers/cakephp.rb b/lib/docs/scrapers/cakephp.rb index b123ab7a..6291b4ab 100644 --- a/lib/docs/scrapers/cakephp.rb +++ b/lib/docs/scrapers/cakephp.rb @@ -71,10 +71,9 @@ module Docs self.base_url = 'https://api.cakephp.org/2.7/' end - def get_latest_version(options, &block) - fetch_doc('https://api.cakephp.org/3.7/', options) do |doc| - block.call doc.at_css('.version-picker .dropdown-toggle').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://api.cakephp.org/3.7/', opts) + doc.at_css('.version-picker .dropdown-toggle').content.strip end private diff --git a/lib/docs/scrapers/chai.rb b/lib/docs/scrapers/chai.rb index 422bd5a9..759f7540 100644 --- a/lib/docs/scrapers/chai.rb +++ b/lib/docs/scrapers/chai.rb @@ -24,8 +24,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('chai', options, &block) + def get_latest_version(opts) + get_npm_version('chai', opts) end end end diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb index f9a248bd..f0b7d6b0 100644 --- a/lib/docs/scrapers/chef.rb +++ b/lib/docs/scrapers/chef.rb @@ -48,10 +48,9 @@ module Docs options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//] end - def get_latest_version(options, &block) - fetch_doc('https://downloads.chef.io/chef', options) do |doc| - block.call doc.at_css('h1.product-heading > span').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://downloads.chef.io/chef', opts) + doc.at_css('h1.product-heading > span').content.strip end end end diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb index 465a4493..b5785bd3 100644 --- a/lib/docs/scrapers/clojure.rb +++ b/lib/docs/scrapers/clojure.rb @@ -28,10 +28,9 @@ module Docs self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/' end - def get_latest_version(options, &block) - fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc| - block.call doc.at_css('#header-version').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://clojure.github.io/clojure/index.html', opts) + doc.at_css('#header-version').content[1..-1] end end end diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb index dde4721c..7548a4a9 100644 --- a/lib/docs/scrapers/cmake.rb +++ b/lib/docs/scrapers/cmake.rb @@ -60,11 +60,10 @@ module Docs self.base_url = 'https://cmake.org/cmake/help/v3.5/' end - def get_latest_version(options, &block) - fetch_doc('https://cmake.org/documentation/', options) do |doc| - link = doc.at_css('.entry-content ul > li > strong > a > big') - block.call link.content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://cmake.org/documentation/', opts) + link = doc.at_css('.entry-content ul > li > strong > a > big') + link.content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb index 2e28de7f..caafc9cf 100644 --- a/lib/docs/scrapers/codeception.rb +++ b/lib/docs/scrapers/codeception.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://codeception.com/changelog', options) do |doc| - block.call doc.at_css('#page > h4').content - end + def get_latest_version(opts) + doc = fetch_doc('https://codeception.com/changelog', opts) + doc.at_css('#page > h4').content end end end diff --git a/lib/docs/scrapers/codeceptjs.rb b/lib/docs/scrapers/codeceptjs.rb index e3f4fda8..34d9b855 100644 --- a/lib/docs/scrapers/codeceptjs.rb +++ b/lib/docs/scrapers/codeceptjs.rb @@ -22,8 +22,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('codeceptjs', options, &block) + def get_latest_version(opts) + get_npm_version('codeceptjs', opts) end end end diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb index 864cf700..05258d9a 100644 --- a/lib/docs/scrapers/codeigniter.rb +++ b/lib/docs/scrapers/codeigniter.rb @@ -39,11 +39,10 @@ module Docs self.release = '3.1.8' end - def get_latest_version(options, &block) - fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc| - header = doc.at_css('#change-log h2') - block.call header.content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://codeigniter.com/user_guide/changelog.html', opts) + header = doc.at_css('#change-log h2') + header.content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb index d848d208..695f3697 100644 --- a/lib/docs/scrapers/coffeescript.rb +++ b/lib/docs/scrapers/coffeescript.rb @@ -31,8 +31,8 @@ module Docs options[:container] = '.container' end - def get_latest_version(options, &block) - get_npm_version('coffeescript', options, &block) + def get_latest_version(opts) + get_npm_version('coffeescript', opts) end end end diff --git a/lib/docs/scrapers/cordova.rb b/lib/docs/scrapers/cordova.rb index efe8fb03..65cf7f60 100644 --- a/lib/docs/scrapers/cordova.rb +++ b/lib/docs/scrapers/cordova.rb @@ -43,13 +43,14 @@ module Docs self.base_url = 'https://cordova.apache.org/docs/en/6.x/' end - def get_latest_version(options, &block) - fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc| - label = doc.at_css('#versionDropdown').content.strip - version = label.scan(/([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://cordova.apache.org/docs/en/latest/', opts) + + label = doc.at_css('#versionDropdown').content.strip + version = label.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end end end diff --git a/lib/docs/scrapers/cpp.rb b/lib/docs/scrapers/cpp.rb index d26eae6a..f96ee8f1 100644 --- a/lib/docs/scrapers/cpp.rb +++ b/lib/docs/scrapers/cpp.rb @@ -35,12 +35,11 @@ module Docs HTML # Same as get_latest_version in lib/docs/scrapers/c.rb - def get_latest_version(options, &block) - fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| - link = doc.at_css('a[title^="File:"]') - date = link.content.scan(/(\d+)\./)[0][0] - block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i - end + def get_latest_version(opts) + doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts) + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + DateTime.strptime(date, '%Y%m%d').to_time.to_i end private diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb index e70317f2..14537c7f 100644 --- a/lib/docs/scrapers/crystal.rb +++ b/lib/docs/scrapers/crystal.rb @@ -35,10 +35,9 @@ module Docs end } - def get_latest_version(options, &block) - fetch('https://crystal-lang.org/api', options) do |body| - block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://crystal-lang.org/api', opts) + body.scan(/Crystal Docs ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/d.rb b/lib/docs/scrapers/d.rb index b0adaf31..e1475b45 100644 --- a/lib/docs/scrapers/d.rb +++ b/lib/docs/scrapers/d.rb @@ -27,10 +27,9 @@ module Docs %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html) end - def get_latest_version(options, &block) - fetch_doc('https://dlang.org/changelog/', options) do |doc| - block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id'] - end + def get_latest_version(opts) + doc = fetch_doc('https://dlang.org/changelog/', opts) + doc.at_css('#content > ul > li:nth-child(2) > a')['id'] end end end diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb index cfbbafc9..e26c1f3d 100644 --- a/lib/docs/scrapers/d3.rb +++ b/lib/docs/scrapers/d3.rb @@ -59,8 +59,8 @@ module Docs options[:only_patterns] = [/\.md\z/] end - def get_latest_version(options, &block) - get_npm_version('d3', options, &block) + def get_latest_version(opts) + get_npm_version('d3', opts) end end end diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb index 42d20423..322bfe2a 100644 --- a/lib/docs/scrapers/dart.rb +++ b/lib/docs/scrapers/dart.rb @@ -32,11 +32,10 @@ module Docs self.base_url = "https://api.dartlang.org/stable/#{release}/" end - def get_latest_version(options, &block) - fetch_doc('https://api.dartlang.org/', options) do |doc| - label = doc.at_css('footer > span').content.strip - block.call label.sub(/Dart /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://api.dartlang.org/', opts) + label = doc.at_css('footer > span').content.strip + label.sub(/Dart /, '') end end end diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb index 746c0f40..6d48c6d7 100644 --- a/lib/docs/scrapers/django.rb +++ b/lib/docs/scrapers/django.rb @@ -64,10 +64,9 @@ module Docs self.base_url = 'https://docs.djangoproject.com/en/1.8/' end - def get_latest_version(options, &block) - fetch_doc('https://docs.djangoproject.com/', options) do |doc| - block.call doc.at_css('#doc-versions > li.current > span > strong').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.djangoproject.com/', opts) + doc.at_css('#doc-versions > li.current > span > strong').content end end end diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb index dd849391..3ef60aab 100644 --- a/lib/docs/scrapers/docker.rb +++ b/lib/docs/scrapers/docker.rb @@ -138,11 +138,10 @@ module Docs options[:only_patterns] << /\Aswarm\// end - def get_latest_version(options, &block) - fetch_doc('https://docs.docker.com/', options) do |doc| - label = doc.at_css('.nav-container button.dropdown-toggle').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.docker.com/', opts) + label = doc.at_css('.nav-container button.dropdown-toggle').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb index 66dccb6f..79898916 100644 --- a/lib/docs/scrapers/dojo.rb +++ b/lib/docs/scrapers/dojo.rb @@ -36,10 +36,9 @@ module Docs urls.map { |url| "#{url}" }.join end - def get_latest_version(options, &block) - fetch_doc('https://dojotoolkit.org/api/', options) do |doc| - block.call doc.at_css('#versionSelector > option[selected]').content - end + def get_latest_version(opts) + doc = fetch_doc('https://dojotoolkit.org/api/', opts) + doc.at_css('#versionSelector > option[selected]').content end private diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb index 92da4193..f29b585b 100644 --- a/lib/docs/scrapers/drupal.rb +++ b/lib/docs/scrapers/drupal.rb @@ -99,13 +99,14 @@ module Docs ] end - def get_latest_version(options, &block) - fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc| - version = doc.at_css('td.form > form > select > option[selected]').content - version = version.scan(/([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('http://cgit.drupalcode.org/drupal', opts) + + version = doc.at_css('td.form > form > select > option[selected]').content + version = version.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end end end diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb index dd3cf00a..8e635f49 100644 --- a/lib/docs/scrapers/electron.rb +++ b/lib/docs/scrapers/electron.rb @@ -23,10 +23,9 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://electronjs.org/docs', options) do |doc| - block.call doc.at_css('.docs-version').content - end + def get_latest_version(opts) + doc = fetch_doc('https://electronjs.org/docs', opts) + doc.at_css('.docs-version').content end end end diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index d5b8dbe6..a25cca41 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -98,10 +98,9 @@ module Docs ] end - def get_latest_version(options, &block) - fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc| - block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts) + doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/ember.rb b/lib/docs/scrapers/ember.rb index 24a8817e..6f853bb9 100644 --- a/lib/docs/scrapers/ember.rb +++ b/lib/docs/scrapers/ember.rb @@ -57,10 +57,9 @@ module Docs ) end - def get_latest_version(options, &block) - fetch_doc('https://emberjs.com/api/ember/release', options) do |doc| - block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://emberjs.com/api/ember/release', opts) + doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip end end end diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb index 0211da55..14a87cf5 100644 --- a/lib/docs/scrapers/erlang.rb +++ b/lib/docs/scrapers/erlang.rb @@ -56,10 +56,9 @@ module Docs self.release = '18.3' end - def get_latest_version(options, &block) - fetch_doc('https://www.erlang.org/downloads', options) do |doc| - block.call doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://www.erlang.org/downloads', opts) + doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '') end end end diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb index dac9c283..c213eacc 100644 --- a/lib/docs/scrapers/eslint.rb +++ b/lib/docs/scrapers/eslint.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('eslint', options, &block) + def get_latest_version(opts) + get_npm_version('eslint', opts) end end end diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb index 67ba07e8..990019fb 100644 --- a/lib/docs/scrapers/express.rb +++ b/lib/docs/scrapers/express.rb @@ -29,8 +29,8 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License v3.0. HTML - def get_latest_version(options, &block) - get_npm_version('express', options, &block) + def get_latest_version(opts) + get_npm_version('express', opts) end end end diff --git a/lib/docs/scrapers/falcon.rb b/lib/docs/scrapers/falcon.rb index cd5b70cd..8ba69150 100644 --- a/lib/docs/scrapers/falcon.rb +++ b/lib/docs/scrapers/falcon.rb @@ -34,10 +34,9 @@ module Docs self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/" end - def get_latest_version(options, &block) - fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc| - block.call doc.at_css('#changelogs ul > li > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', opts) + doc.at_css('#changelogs ul > li > a').content end end end diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb index 9340961a..c9a98802 100644 --- a/lib/docs/scrapers/fish.rb +++ b/lib/docs/scrapers/fish.rb @@ -47,10 +47,9 @@ module Docs self.base_url = "https://fishshell.com/docs/#{version}/" end - def get_latest_version(options, &block) - fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc| - block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://fishshell.com/docs/current/index.html', opts) + doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb index 546473f7..b3b5a02f 100644 --- a/lib/docs/scrapers/flow.rb +++ b/lib/docs/scrapers/flow.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('flow-bin', options, &block) + def get_latest_version(opts) + get_npm_version('flow-bin', opts) end end end diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb index f10473d0..9de5cb0d 100644 --- a/lib/docs/scrapers/git.rb +++ b/lib/docs/scrapers/git.rb @@ -20,10 +20,9 @@ module Docs Licensed under the GNU General Public License version 2. HTML - def get_latest_version(options, &block) - fetch_doc('https://git-scm.com/', options) do |doc| - block.call doc.at_css('.version').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://git-scm.com/', opts) + doc.at_css('.version').content.strip end end end diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb index 3252dd6d..565706d9 100644 --- a/lib/docs/scrapers/gnu/gcc.rb +++ b/lib/docs/scrapers/gnu/gcc.rb @@ -100,11 +100,10 @@ module Docs options[:replace_paths] = CPP_PATHS end - def get_latest_version(options, &block) - fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| - label = doc.at_css('ul > li > ul > li > a').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts) + label = doc.at_css('ul > li > ul > li > a').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb index f72f7d65..dd18827c 100644 --- a/lib/docs/scrapers/gnu/gnu_fortran.rb +++ b/lib/docs/scrapers/gnu/gnu_fortran.rb @@ -26,11 +26,10 @@ module Docs self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/" end - def get_latest_version(options, &block) - fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| - label = doc.at_css('ul > li > ul > li > a').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts) + label = doc.at_css('ul > li > ul > li > a').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb index 6f8f7a4a..8997b7b1 100644 --- a/lib/docs/scrapers/go.rb +++ b/lib/docs/scrapers/go.rb @@ -24,13 +24,14 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://golang.org/pkg/', options) do |doc| - footer = doc.at_css('#footer').content - version = footer.scan(/go([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://golang.org/pkg/', opts) + + footer = doc.at_css('#footer').content + version = footer.scan(/go([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end private diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb index d43782c2..06c330b2 100644 --- a/lib/docs/scrapers/godot.rb +++ b/lib/docs/scrapers/godot.rb @@ -38,10 +38,9 @@ module Docs self.base_url = "http://docs.godotengine.org/en/#{self.version}/" end - def get_latest_version(options, &block) - fetch_doc('https://docs.godotengine.org/', options) do |doc| - block.call doc.at_css('.version').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.godotengine.org/', opts) + doc.at_css('.version').content.strip end end end diff --git a/lib/docs/scrapers/graphite.rb b/lib/docs/scrapers/graphite.rb index d1d8b9d1..83e9314a 100644 --- a/lib/docs/scrapers/graphite.rb +++ b/lib/docs/scrapers/graphite.rb @@ -18,10 +18,9 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc| - block.call doc.at_css('#release-notes li > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', opts) + doc.at_css('#release-notes li > a').content end end end diff --git a/lib/docs/scrapers/grunt.rb b/lib/docs/scrapers/grunt.rb index 1e8af9fb..469d10a0 100644 --- a/lib/docs/scrapers/grunt.rb +++ b/lib/docs/scrapers/grunt.rb @@ -27,8 +27,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('grunt-cli', options, &block) + def get_latest_version(opts) + get_npm_version('grunt-cli', opts) end end end diff --git a/lib/docs/scrapers/handlebars.rb b/lib/docs/scrapers/handlebars.rb index 7df63102..046cdf0f 100644 --- a/lib/docs/scrapers/handlebars.rb +++ b/lib/docs/scrapers/handlebars.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('handlebars', options, &block) + def get_latest_version(opts) + get_npm_version('handlebars', opts) end end end diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index fc848a7a..fb118851 100755 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -69,12 +69,11 @@ module Docs options[:only_patterns] = [/\Alibraries\//] end - def get_latest_version(options, &block) - fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', options) do |doc| - links = doc.css('a').to_a - versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)} - block.call versions.find {|version| !version.empty?}[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', opts) + links = doc.css('a').to_a + versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)} + versions.find {|version| !version.empty?}[0][0] end end end diff --git a/lib/docs/scrapers/haxe.rb b/lib/docs/scrapers/haxe.rb index 5a685efc..2dbab01a 100644 --- a/lib/docs/scrapers/haxe.rb +++ b/lib/docs/scrapers/haxe.rb @@ -67,11 +67,10 @@ module Docs self.base_url = 'https://api.haxe.org/python/' end - def get_latest_version(options, &block) - fetch_doc('https://api.haxe.org/', options) do |doc| - label = doc.at_css('.container.main-content h1 > small').content - block.call label.sub(/version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://api.haxe.org/', opts) + label = doc.at_css('.container.main-content h1 > small').content + label.sub(/version /, '') end end end diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index fef1ed05..c5647709 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -20,10 +20,8 @@ module Docs Licensed under the BSD 2-Clause License. HTML - def get_latest_version(options, &block) - get_latest_github_release('Homebrew', 'brew', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('Homebrew', 'brew', opts)['name'] end end end diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb index 342ce107..8b1b47a2 100644 --- a/lib/docs/scrapers/immutable.rb +++ b/lib/docs/scrapers/immutable.rb @@ -55,8 +55,8 @@ module Docs capybara.html end - def get_latest_version(options, &block) - get_npm_version('immutable', options, &block) + def get_latest_version(opts) + get_npm_version('immutable', opts) end end end diff --git a/lib/docs/scrapers/influxdata.rb b/lib/docs/scrapers/influxdata.rb index 4fc98c16..db160f9c 100644 --- a/lib/docs/scrapers/influxdata.rb +++ b/lib/docs/scrapers/influxdata.rb @@ -47,11 +47,10 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc| - label = doc.at_css('.navbar--current-product').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.influxdata.com/influxdb/', opts) + label = doc.at_css('.navbar--current-product').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb index 5f38e3d5..14c51869 100644 --- a/lib/docs/scrapers/jasmine.rb +++ b/lib/docs/scrapers/jasmine.rb @@ -18,10 +18,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('jasmine', 'jasmine', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('jasmine', 'jasmine', opts)['name'] end end end diff --git a/lib/docs/scrapers/jekyll.rb b/lib/docs/scrapers/jekyll.rb index a6af352f..500eee10 100644 --- a/lib/docs/scrapers/jekyll.rb +++ b/lib/docs/scrapers/jekyll.rb @@ -29,10 +29,9 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://jekyllrb.com/docs/', options) do |doc| - block.call doc.at_css('.meta a').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://jekyllrb.com/docs/', opts) + doc.at_css('.meta a').content[1..-1] end end end diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb index 71efcf54..a495d939 100644 --- a/lib/docs/scrapers/jest.rb +++ b/lib/docs/scrapers/jest.rb @@ -18,10 +18,9 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc| - block.call doc.at_css('header > a > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://jestjs.io/docs/en/getting-started', opts) + doc.at_css('header > a > h3').content end end end diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb index dad609e7..a0c8b97a 100644 --- a/lib/docs/scrapers/jquery/jquery_core.rb +++ b/lib/docs/scrapers/jquery/jquery_core.rb @@ -23,8 +23,8 @@ module Docs /index/i ] - def get_latest_version(options, &block) - get_npm_version('jquery', options, &block) + def get_latest_version(opts) + get_npm_version('jquery', opts) end end end diff --git a/lib/docs/scrapers/jquery/jquery_mobile.rb b/lib/docs/scrapers/jquery/jquery_mobile.rb index 53b2c624..5b856a95 100644 --- a/lib/docs/scrapers/jquery/jquery_mobile.rb +++ b/lib/docs/scrapers/jquery/jquery_mobile.rb @@ -17,11 +17,9 @@ module Docs url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/' end - def get_latest_version(options, &block) - fetch_doc('https://jquerymobile.com/', options) do |doc| - label = doc.at_css('.download-box > .download-option:last-child > span').content - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://jquerymobile.com/', opts) + doc.at_css('.download-box > .download-option:last-child > span').content.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb index 05c276e1..021d1d22 100644 --- a/lib/docs/scrapers/jquery/jquery_ui.rb +++ b/lib/docs/scrapers/jquery/jquery_ui.rb @@ -16,8 +16,8 @@ module Docs url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/' end - def get_latest_version(options, &block) - get_npm_version('jquery-ui', options, &block) + def get_latest_version(opts) + get_npm_version('jquery-ui', opts) end end end diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb index 39feca71..d88d46b6 100644 --- a/lib/docs/scrapers/jsdoc.rb +++ b/lib/docs/scrapers/jsdoc.rb @@ -22,10 +22,8 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('jsdoc3', 'jsdoc', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('jsdoc3', 'jsdoc', opts)['tag_name'] end end end diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb index 0875835a..d152f05d 100644 --- a/lib/docs/scrapers/julia.rb +++ b/lib/docs/scrapers/julia.rb @@ -50,10 +50,8 @@ module Docs html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html' end - def get_latest_version(options, &block) - get_latest_github_release('JuliaLang', 'julia', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('JuliaLang', 'julia', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb index 60af1540..6556bca8 100644 --- a/lib/docs/scrapers/knockout.rb +++ b/lib/docs/scrapers/knockout.rb @@ -34,10 +34,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('knockout', 'knockout', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('knockout', 'knockout', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/koa.rb b/lib/docs/scrapers/koa.rb index 4d90e30f..cac14920 100644 --- a/lib/docs/scrapers/koa.rb +++ b/lib/docs/scrapers/koa.rb @@ -35,8 +35,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('koa', options, &block) + def get_latest_version(opts) + get_npm_version('koa', opts) end end end diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb index 7539212d..5f508ae7 100644 --- a/lib/docs/scrapers/kotlin.rb +++ b/lib/docs/scrapers/kotlin.rb @@ -29,10 +29,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('JetBrains', 'kotlin', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('JetBrains', 'kotlin', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index cdf32732..4fc17368 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -134,10 +134,8 @@ module Docs end end - def get_latest_version(options, &block) - get_latest_github_release('laravel', 'laravel', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('laravel', 'laravel', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb index 24bc6142..38e497e7 100644 --- a/lib/docs/scrapers/leaflet.rb +++ b/lib/docs/scrapers/leaflet.rb @@ -39,11 +39,10 @@ module Docs self.base_url = "https://leafletjs.com/reference-#{release}.html" end - def get_latest_version(options, &block) - fetch_doc('https://leafletjs.com/index.html', options) do |doc| - link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first - block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://leafletjs.com/index.html', opts) + link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first + link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] end end end diff --git a/lib/docs/scrapers/less.rb b/lib/docs/scrapers/less.rb index 00c884eb..b19bbe17 100644 --- a/lib/docs/scrapers/less.rb +++ b/lib/docs/scrapers/less.rb @@ -22,11 +22,10 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML - def get_latest_version(options, &block) - fetch_doc('http://lesscss.org/features/', options) do |doc| - label = doc.at_css('.footer-links > li').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://lesscss.org/features/', opts) + label = doc.at_css('.footer-links > li').content + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/liquid.rb b/lib/docs/scrapers/liquid.rb index 4630b2d1..b8e40d59 100644 --- a/lib/docs/scrapers/liquid.rb +++ b/lib/docs/scrapers/liquid.rb @@ -20,10 +20,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_tags('Shopify', 'liquid', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('Shopify', 'liquid', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb index 5488b9ab..bce625e6 100644 --- a/lib/docs/scrapers/lodash.rb +++ b/lib/docs/scrapers/lodash.rb @@ -33,10 +33,9 @@ module Docs self.base_url = "https://lodash.com/docs/#{release}" end - def get_latest_version(options, &block) - fetch_doc('https://lodash.com/docs/', options) do |doc| - block.call doc.at_css('#version > option[selected]').content - end + def get_latest_version(opts) + doc = fetch_doc('https://lodash.com/docs/', opts) + doc.at_css('#version > option[selected]').content end end end diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb index 019edbab..887b796f 100644 --- a/lib/docs/scrapers/love.rb +++ b/lib/docs/scrapers/love.rb @@ -40,10 +40,9 @@ module Docs Licensed under the GNU Free Documentation License, Version 1.3. HTML - def get_latest_version(options, &block) - fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc| - block.call doc.at_css('#mw-content-text table a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://love2d.org/wiki/Version_History', opts) + doc.at_css('#mw-content-text table a').content end end end diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb index 30af5523..e3608918 100644 --- a/lib/docs/scrapers/lua.rb +++ b/lib/docs/scrapers/lua.rb @@ -27,10 +27,9 @@ module Docs self.base_url = 'https://www.lua.org/manual/5.1/' end - def get_latest_version(options, &block) - fetch_doc('https://www.lua.org/manual/', options) do |doc| - block.call doc.at_css('p.menubar > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://www.lua.org/manual/', opts) + doc.at_css('p.menubar > a').content end end end diff --git a/lib/docs/scrapers/marionette.rb b/lib/docs/scrapers/marionette.rb index 12de6d0c..fd1eab8e 100644 --- a/lib/docs/scrapers/marionette.rb +++ b/lib/docs/scrapers/marionette.rb @@ -39,8 +39,8 @@ module Docs html_filters.push 'marionette/entries_v2' end - def get_latest_version(options, &block) - get_npm_version('backbone.marionette', options, &block) + def get_latest_version(opts) + get_npm_version('backbone.marionette', opts) end end end diff --git a/lib/docs/scrapers/markdown.rb b/lib/docs/scrapers/markdown.rb index 3400e270..b837c692 100644 --- a/lib/docs/scrapers/markdown.rb +++ b/lib/docs/scrapers/markdown.rb @@ -14,8 +14,8 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - block.call '1.0.0' + def get_latest_version(opts) + '1.0.0' end end end diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb index 948955a6..4a882270 100644 --- a/lib/docs/scrapers/matplotlib.rb +++ b/lib/docs/scrapers/matplotlib.rb @@ -65,10 +65,8 @@ module Docs ] end - def get_latest_version(options, &block) - get_latest_github_release('matplotlib', 'matplotlib', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('matplotlib', 'matplotlib', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb index ccb27af9..defb4533 100644 --- a/lib/docs/scrapers/mdn/mdn.rb +++ b/lib/docs/scrapers/mdn/mdn.rb @@ -21,10 +21,9 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later. HTML - def get_latest_version(opts, &block) - fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts) do |json| - block.call DateTime.parse(json[0]['pubdate']).to_time.to_i - end + def get_latest_version(opts) + json = fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts) + DateTime.parse(json[0]['pubdate']).to_time.to_i end private diff --git a/lib/docs/scrapers/meteor.rb b/lib/docs/scrapers/meteor.rb index 02b81bc3..a758d154 100644 --- a/lib/docs/scrapers/meteor.rb +++ b/lib/docs/scrapers/meteor.rb @@ -46,10 +46,9 @@ module Docs options[:fix_urls] = nil end - def get_latest_version(options, &block) - fetch_doc('https://docs.meteor.com/#/full/', options) do |doc| - block.call doc.at_css('select.version-select > option').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.meteor.com/#/full/', opts) + doc.at_css('select.version-select > option').content end end end diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb index 6654d754..04945425 100644 --- a/lib/docs/scrapers/mocha.rb +++ b/lib/docs/scrapers/mocha.rb @@ -19,8 +19,8 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International License. HTML - def get_latest_version(options, &block) - get_npm_version('mocha', options, &block) + def get_latest_version(opts) + get_npm_version('mocha', opts) end end end diff --git a/lib/docs/scrapers/modernizr.rb b/lib/docs/scrapers/modernizr.rb index 93a738bb..01ad49a7 100644 --- a/lib/docs/scrapers/modernizr.rb +++ b/lib/docs/scrapers/modernizr.rb @@ -16,8 +16,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('modernizr', options, &block) + def get_latest_version(opts) + get_npm_version('modernizr', opts) end end end diff --git a/lib/docs/scrapers/moment.rb b/lib/docs/scrapers/moment.rb index 9dd27107..5b7491ea 100644 --- a/lib/docs/scrapers/moment.rb +++ b/lib/docs/scrapers/moment.rb @@ -23,10 +23,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('http://momentjs.com/', options) do |doc| - block.call doc.at_css('.hero-title > h1 > span').content - end + def get_latest_version(opts) + doc = fetch_doc('http://momentjs.com/', opts) + doc.at_css('.hero-title > h1 > span').content end end end diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb index 2d221990..fbd4ca92 100644 --- a/lib/docs/scrapers/mongoose.rb +++ b/lib/docs/scrapers/mongoose.rb @@ -27,11 +27,10 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://mongoosejs.com/docs/', options) do |doc| - label = doc.at_css('.pure-menu-link').content.strip - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://mongoosejs.com/docs/', opts) + label = doc.at_css('.pure-menu-link').content.strip + label.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb index 07a4565f..5354b8bd 100644 --- a/lib/docs/scrapers/nginx.rb +++ b/lib/docs/scrapers/nginx.rb @@ -26,11 +26,10 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://nginx.org/en/download.html', options) do |doc| - table = doc.at_css('#content > table').inner_html - block.call table.scan(/nginx-([0-9.]+) table').inner_html + table.scan(/nginx-([0-9.]+) .docinfo > tbody > tr:last-child > td').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://nim-lang.org/docs/overview.html', opts) + doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip end end end diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb index 36c14e9e..0e5ee8fa 100644 --- a/lib/docs/scrapers/node.rb +++ b/lib/docs/scrapers/node.rb @@ -47,10 +47,9 @@ module Docs self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/' end - def get_latest_version(options, &block) - fetch_doc('https://nodejs.org/en/', options) do |doc| - block.call doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://nodejs.org/en/', opts) + doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1] end end end diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb index aca5d10c..9da5daf4 100644 --- a/lib/docs/scrapers/nokogiri2.rb +++ b/lib/docs/scrapers/nokogiri2.rb @@ -20,10 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('sparklemotion', 'nokogiri', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('sparklemotion', 'nokogiri', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb index 7ef727c0..e18531ab 100644 --- a/lib/docs/scrapers/npm.rb +++ b/lib/docs/scrapers/npm.rb @@ -30,10 +30,8 @@ module Docs npm is a trademark of npm, Inc. HTML - def get_latest_version(options, &block) - get_latest_github_release('npm', 'cli', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('npm', 'cli', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb index 1fcc05d6..636fcf4a 100644 --- a/lib/docs/scrapers/numpy.rb +++ b/lib/docs/scrapers/numpy.rb @@ -50,10 +50,8 @@ module Docs self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/" end - def get_latest_version(options, &block) - get_latest_github_release('numpy', 'numpy', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('numpy', 'numpy', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb index a56a2928..c26bce4c 100644 --- a/lib/docs/scrapers/openjdk.rb +++ b/lib/docs/scrapers/openjdk.rb @@ -87,7 +87,7 @@ module Docs File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil end - def get_latest_version(options, &block) + def get_latest_version(opts) latest_version = 8 current_attempt = latest_version attempts = 0 @@ -95,17 +95,16 @@ module Docs while attempts < 3 current_attempt += 1 - fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", options) do |doc| - if doc.at_css('.perror').nil? - latest_version = current_attempt - attempts = 0 - else - attempts += 1 - end + doc = fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", opts) + if doc.at_css('.perror').nil? + latest_version = current_attempt + attempts = 0 + else + attempts += 1 end end - block.call latest_version + latest_version end end end diff --git a/lib/docs/scrapers/opentsdb.rb b/lib/docs/scrapers/opentsdb.rb index 54a77017..1de40478 100644 --- a/lib/docs/scrapers/opentsdb.rb +++ b/lib/docs/scrapers/opentsdb.rb @@ -19,10 +19,8 @@ module Docs Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses. HTML - def get_latest_version(options, &block) - get_latest_github_release('OpenTSDB', 'opentsdb', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('OpenTSDB', 'opentsdb', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/padrino.rb b/lib/docs/scrapers/padrino.rb index 78588ef4..d34b7db5 100644 --- a/lib/docs/scrapers/padrino.rb +++ b/lib/docs/scrapers/padrino.rb @@ -24,10 +24,8 @@ module Docs request_one(url_for('index')).body end - def get_latest_version(options, &block) - get_github_tags('padrino', 'padrino-framework', options) do |tags| - block.call tags[0]['name'] - end + def get_latest_version(opts) + get_github_tags('padrino', 'padrino-framework', opts)[0]['name'] end end end diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb index f50c4427..a8c2ea51 100644 --- a/lib/docs/scrapers/pandas.rb +++ b/lib/docs/scrapers/pandas.rb @@ -50,11 +50,10 @@ module Docs self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/" end - def get_latest_version(options, &block) - fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', options) do |doc| - label = doc.at_css('.body > .section > p').content - block.call label.scan(/Version: ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', opts) + label = doc.at_css('.body > .section > p').content + label.scan(/Version: ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb index 136a6d75..7e5ed8f8 100644 --- a/lib/docs/scrapers/perl.rb +++ b/lib/docs/scrapers/perl.rb @@ -44,10 +44,9 @@ module Docs self.base_url = "https://perldoc.perl.org/#{self.release}/" end - def get_latest_version(options, &block) - fetch('https://perldoc.perl.org/static/perlversion.js', options) do |body| - block.call body.scan(/>Perl ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://perldoc.perl.org/static/perlversion.js', opts) + body.scan(/>Perl ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/phalcon.rb b/lib/docs/scrapers/phalcon.rb index 82a9cf85..dd476329 100644 --- a/lib/docs/scrapers/phalcon.rb +++ b/lib/docs/scrapers/phalcon.rb @@ -30,10 +30,9 @@ module Docs self.base_url = 'https://docs.phalconphp.com/en/2.0.0/' end - def get_latest_version(options, &block) - fetch_doc('https://docs.phalconphp.com/', options) do |doc| - block.call doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.phalconphp.com/', opts) + doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/phaser.rb b/lib/docs/scrapers/phaser.rb index ef265e2e..1939b1d0 100644 --- a/lib/docs/scrapers/phaser.rb +++ b/lib/docs/scrapers/phaser.rb @@ -26,10 +26,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('photonstorm', 'phaser', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('photonstorm', 'phaser', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/phoenix.rb b/lib/docs/scrapers/phoenix.rb index 08948b3c..2ad053e2 100644 --- a/lib/docs/scrapers/phoenix.rb +++ b/lib/docs/scrapers/phoenix.rb @@ -47,10 +47,9 @@ module Docs end } - def get_latest_version(options, &block) - fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', options) do |doc| - block.call doc.at_css('.sidebar-projectVersion').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', opts) + doc.at_css('.sidebar-projectVersion').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb index b6aac039..181d8b67 100644 --- a/lib/docs/scrapers/php.rb +++ b/lib/docs/scrapers/php.rb @@ -67,11 +67,10 @@ module Docs Licensed under the Creative Commons Attribution License v3.0 or later. HTML - def get_latest_version(options, &block) - fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', options) do |doc| - label = doc.at_css('tbody.gen-changelog > tr > td').content - block.call label.split(',').last.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', opts) + label = doc.at_css('tbody.gen-changelog > tr > td').content + label.split(',').last.strip end end end diff --git a/lib/docs/scrapers/phpunit.rb b/lib/docs/scrapers/phpunit.rb index c5e8c396..5f2cbfea 100644 --- a/lib/docs/scrapers/phpunit.rb +++ b/lib/docs/scrapers/phpunit.rb @@ -38,11 +38,10 @@ module Docs self.base_url = "https://phpunit.de/manual/#{release}/en/" end - def get_latest_version(options, &block) - fetch_doc('https://phpunit.readthedocs.io/', options) do |doc| - label = doc.at_css('.rst-current-version').content.strip - block.call label.scan(/v: ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://phpunit.readthedocs.io/', opts) + label = doc.at_css('.rst-current-version').content.strip + label.scan(/v: ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb index 4946066b..cc7a85c8 100644 --- a/lib/docs/scrapers/postgresql.rb +++ b/lib/docs/scrapers/postgresql.rb @@ -81,11 +81,10 @@ module Docs html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names' end - def get_latest_version(options, &block) - fetch_doc('https://www.postgresql.org/docs/current/index.html', options) do |doc| - label = doc.at_css('#pgContentWrap h1.title').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://www.postgresql.org/docs/current/index.html', opts) + label = doc.at_css('#pgContentWrap h1.title').content + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/pug.rb b/lib/docs/scrapers/pug.rb index 9084feb1..79a1b0a8 100644 --- a/lib/docs/scrapers/pug.rb +++ b/lib/docs/scrapers/pug.rb @@ -18,8 +18,8 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - get_npm_version('pug', options, &block) + def get_latest_version(opts) + get_npm_version('pug', opts) end private diff --git a/lib/docs/scrapers/puppeteer.rb b/lib/docs/scrapers/puppeteer.rb index a1506c7a..043380d5 100644 --- a/lib/docs/scrapers/puppeteer.rb +++ b/lib/docs/scrapers/puppeteer.rb @@ -15,10 +15,9 @@ module Docs Licensed under the Apache License 2.0. HTML - def get_latest_version(options, &block) - get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', options) do |contents| - block.call contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', opts) + contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0] end end end diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb index 538c0722..d5a5581d 100644 --- a/lib/docs/scrapers/pygame.rb +++ b/lib/docs/scrapers/pygame.rb @@ -18,10 +18,8 @@ module Docs Licensed under the GNU LGPL License version 2.1. HTML - def get_latest_version(options, &block) - get_latest_github_release('pygame', 'pygame', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('pygame', 'pygame', opts)['tag_name'] end end end diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb index 16ecc366..c7905591 100644 --- a/lib/docs/scrapers/python.rb +++ b/lib/docs/scrapers/python.rb @@ -51,10 +51,9 @@ module Docs html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html' end - def get_latest_version(options, &block) - fetch_doc('https://docs.python.org/', options) do |doc| - block.call doc.at_css('.version_switcher_placeholder').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.python.org/', opts) + doc.at_css('.version_switcher_placeholder').content end end end diff --git a/lib/docs/scrapers/q.rb b/lib/docs/scrapers/q.rb index 0a160488..a4c449c0 100644 --- a/lib/docs/scrapers/q.rb +++ b/lib/docs/scrapers/q.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('q', options, &block) + def get_latest_version(opts) + get_npm_version('q', opts) end end end diff --git a/lib/docs/scrapers/qt.rb b/lib/docs/scrapers/qt.rb index 41c835e9..a1098b20 100644 --- a/lib/docs/scrapers/qt.rb +++ b/lib/docs/scrapers/qt.rb @@ -118,11 +118,9 @@ module Docs self.base_url = 'https://doc.qt.io/qt-5.6/' end - def get_latest_version(options, &block) - fetch_doc('https://doc.qt.io/qt-5/index.html', options) do |doc| - label = doc.at_css('.mainContent h1.title').content - block.call label.sub(/Qt /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://doc.qt.io/qt-5/index.html', opts) + doc.at_css('.mainContent h1.title').content.sub(/Qt /, '') end end end diff --git a/lib/docs/scrapers/ramda.rb b/lib/docs/scrapers/ramda.rb index d3751e38..0b86e365 100644 --- a/lib/docs/scrapers/ramda.rb +++ b/lib/docs/scrapers/ramda.rb @@ -16,10 +16,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://ramdajs.com/docs/', options) do |doc| - block.call doc.at_css('.navbar-brand > .version').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://ramdajs.com/docs/', opts) + doc.at_css('.navbar-brand > .version').content[1..-1] end end end diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb index f676010d..2a4249fc 100644 --- a/lib/docs/scrapers/rdoc/minitest.rb +++ b/lib/docs/scrapers/rdoc/minitest.rb @@ -22,10 +22,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents| - block.call contents.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', opts) + contents.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb index 9cb2ab9b..771c2f64 100644 --- a/lib/docs/scrapers/rdoc/rails.rb +++ b/lib/docs/scrapers/rdoc/rails.rb @@ -94,10 +94,8 @@ module Docs self.release = '4.1.16' end - def get_latest_version(options, &block) - get_latest_github_release('rails', 'rails', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('rails', 'rails', opts)['name'] end end end diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb index 292540db..bc064660 100644 --- a/lib/docs/scrapers/rdoc/ruby.rb +++ b/lib/docs/scrapers/rdoc/ruby.rb @@ -85,14 +85,13 @@ module Docs self.release = '2.2.10' end - def get_latest_version(options, &block) - get_github_tags('ruby', 'ruby', options) do |tags| - tags.each do |tag| - version = tag['name'].gsub(/_/, '.')[1..-1] - if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 - block.call version - break - end + def get_latest_version(opts) + tags = get_github_tags('ruby', 'ruby', opts) + tags.each do |tag| + version = tag['name'].gsub(/_/, '.')[1..-1] + + if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 + return version end end end diff --git a/lib/docs/scrapers/react.rb b/lib/docs/scrapers/react.rb index 54038244..3c41ea5a 100644 --- a/lib/docs/scrapers/react.rb +++ b/lib/docs/scrapers/react.rb @@ -31,10 +31,9 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International Public License. HTML - def get_latest_version(options, &block) - fetch_doc('https://reactjs.org/docs/getting-started.html', options) do |doc| - block.call doc.at_css('a[href="/versions"]').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://reactjs.org/docs/getting-started.html', opts) + doc.at_css('a[href="/versions"]').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/react_native.rb b/lib/docs/scrapers/react_native.rb index ac5bd250..fe87e492 100644 --- a/lib/docs/scrapers/react_native.rb +++ b/lib/docs/scrapers/react_native.rb @@ -31,10 +31,9 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International Public License. HTML - def get_latest_version(options, &block) - fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', options) do |doc| - block.call doc.at_css('header > a > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', opts) + doc.at_css('header > a > h3').content end end end diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb index 89a67598..81440a82 100644 --- a/lib/docs/scrapers/redis.rb +++ b/lib/docs/scrapers/redis.rb @@ -20,11 +20,10 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License 4.0. HTML - def get_latest_version(options, &block) - fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', options) do |body| - body = body.lines[1..-1].join - block.call body.scan(/Redis ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', opts) + body = body.lines[1..-1].join + body.scan(/Redis ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/redux.rb b/lib/docs/scrapers/redux.rb index 09738bb3..14f8e8b4 100644 --- a/lib/docs/scrapers/redux.rb +++ b/lib/docs/scrapers/redux.rb @@ -21,8 +21,8 @@ module Docs request_one('http://redux.js.org/index.html').body end - def get_latest_version(options, &block) - get_npm_version('redux', options, &block) + def get_latest_version(opts) + get_npm_version('redux', opts) end end end diff --git a/lib/docs/scrapers/relay.rb b/lib/docs/scrapers/relay.rb index 90423ac6..807d6e1d 100644 --- a/lib/docs/scrapers/relay.rb +++ b/lib/docs/scrapers/relay.rb @@ -19,10 +19,9 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('http://facebook.github.io/relay/en/', options) do |doc| - block.call doc.at_css('header > a > h3').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://facebook.github.io/relay/en/', opts) + doc.at_css('header > a > h3').content[1..-1] end end end diff --git a/lib/docs/scrapers/requirejs.rb b/lib/docs/scrapers/requirejs.rb index cf417a3e..80ecc722 100644 --- a/lib/docs/scrapers/requirejs.rb +++ b/lib/docs/scrapers/requirejs.rb @@ -31,8 +31,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('requirejs', options, &block) + def get_latest_version(opts) + get_npm_version('requirejs', opts) end end end diff --git a/lib/docs/scrapers/rethinkdb.rb b/lib/docs/scrapers/rethinkdb.rb index 27d913e0..3a6b87cf 100644 --- a/lib/docs/scrapers/rethinkdb.rb +++ b/lib/docs/scrapers/rethinkdb.rb @@ -58,10 +58,8 @@ module Docs CODE end - def get_latest_version(options, &block) - get_latest_github_release('rethinkdb', 'rethinkdb', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('rethinkdb', 'rethinkdb', opts)['tag_name'][1..-1] end private diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb index 7c8d1519..635afc1c 100644 --- a/lib/docs/scrapers/rust.rb +++ b/lib/docs/scrapers/rust.rb @@ -39,11 +39,10 @@ module Docs Licensed under the Apache License, Version 2.0 or the MIT license, at your option. HTML - def get_latest_version(options, &block) - fetch_doc('https://www.rust-lang.org/', options) do |doc| - label = doc.at_css('.button-download + p > a').content - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://www.rust-lang.org/', opts) + label = doc.at_css('.button-download + p > a').content + label.sub(/Version /, '') end private diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb index f493289b..228a5337 100644 --- a/lib/docs/scrapers/sass.rb +++ b/lib/docs/scrapers/sass.rb @@ -24,10 +24,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_file_contents('sass', 'sass', 'VERSION', options) do |contents| - block.call contents.strip - end + def get_latest_version(opts) + get_github_file_contents('sass', 'sass', 'VERSION', opts).strip end end end diff --git a/lib/docs/scrapers/scikit_image.rb b/lib/docs/scrapers/scikit_image.rb index 3c454bcb..a5959581 100644 --- a/lib/docs/scrapers/scikit_image.rb +++ b/lib/docs/scrapers/scikit_image.rb @@ -21,10 +21,9 @@ module Docs Licensed under the BSD 3-clause License. HTML - def get_latest_version(options, &block) - get_github_tags('scikit-image', 'scikit-image', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('scikit-image', 'scikit-image', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/scikit_learn.rb b/lib/docs/scrapers/scikit_learn.rb index 0d7fc90d..973c9d7e 100644 --- a/lib/docs/scrapers/scikit_learn.rb +++ b/lib/docs/scrapers/scikit_learn.rb @@ -25,11 +25,9 @@ module Docs Licensed under the 3-clause BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://scikit-learn.org/stable/documentation.html', options) do |doc| - label = doc.at_css('.body h1').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://scikit-learn.org/stable/documentation.html', opts) + doc.at_css('.body h1').content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/sinon.rb b/lib/docs/scrapers/sinon.rb index a1bd06cc..632348dc 100644 --- a/lib/docs/scrapers/sinon.rb +++ b/lib/docs/scrapers/sinon.rb @@ -53,10 +53,9 @@ module Docs self.base_url = "https://sinonjs.org/releases/v#{release}/" end - def get_latest_version(options, &block) - fetch('https://sinonjs.org/', options) do |body| - block.call body.scan(/\/releases\/v([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://sinonjs.org/', opts) + body.scan(/\/releases\/v([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/socketio.rb b/lib/docs/scrapers/socketio.rb index 8518ff53..e9bc264a 100644 --- a/lib/docs/scrapers/socketio.rb +++ b/lib/docs/scrapers/socketio.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('socket.io', options, &block) + def get_latest_version(opts) + get_npm_version('socket.io', opts) end end end diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb index 27720111..d3a5fa6b 100644 --- a/lib/docs/scrapers/sqlite.rb +++ b/lib/docs/scrapers/sqlite.rb @@ -41,10 +41,9 @@ module Docs options[:attribution] = 'SQLite is in the Public Domain.' - def get_latest_version(options, &block) - fetch_doc('https://sqlite.org/chronology.html', options) do |doc| - block.call doc.at_css('#chrontab > tbody > tr > td:last-child > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://sqlite.org/chronology.html', opts) + doc.at_css('#chrontab > tbody > tr > td:last-child > a').content end private diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb index 0c9bfd06..db2eacb2 100644 --- a/lib/docs/scrapers/statsmodels.rb +++ b/lib/docs/scrapers/statsmodels.rb @@ -21,10 +21,9 @@ module Docs Licensed under the 3-clause BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('http://www.statsmodels.org/stable/', options) do |doc| - block.call doc.at_css('.sphinxsidebarwrapper h3 + p > b').content - end + def get_latest_version(opts) + doc = fetch_doc('http://www.statsmodels.org/stable/', opts) + doc.at_css('.sphinxsidebarwrapper h3 + p > b').content end end end diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb index 90cd7f4f..d04072da 100644 --- a/lib/docs/scrapers/support_tables.rb +++ b/lib/docs/scrapers/support_tables.rb @@ -179,11 +179,10 @@ module Docs HTML - def get_latest_version(options, &block) - fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', options) do |body| - timestamp = body.scan(/([^<]+)<\/updated>/)[0][0] - block.call DateTime.parse(timestamp).to_time.to_i - end + def get_latest_version(opts) + body = fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', opts) + timestamp = body.scan(/([^<]+)<\/updated>/)[0][0] + DateTime.parse(timestamp).to_time.to_i end end end diff --git a/lib/docs/scrapers/symfony.rb b/lib/docs/scrapers/symfony.rb index acd105da..439456f0 100644 --- a/lib/docs/scrapers/symfony.rb +++ b/lib/docs/scrapers/symfony.rb @@ -71,10 +71,8 @@ module Docs self.base_url = "https://api.symfony.com/#{version}/" end - def get_latest_version(options, &block) - get_latest_github_release('symfony', 'symfony', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('symfony', 'symfony', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/tcl_tk.rb b/lib/docs/scrapers/tcl_tk.rb index 3dcc5765..bca840c6 100644 --- a/lib/docs/scrapers/tcl_tk.rb +++ b/lib/docs/scrapers/tcl_tk.rb @@ -26,11 +26,9 @@ module Docs Licensed under Tcl/Tk terms HTML - def get_latest_version(options, &block) - fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', options) do |doc| - label = doc.at_css('h2').content - block.call label.scan(/Tk([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', opts) + doc.at_css('h2').content.scan(/Tk([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/tensorflow.rb b/lib/docs/scrapers/tensorflow.rb index dfd89875..8ac1ca65 100644 --- a/lib/docs/scrapers/tensorflow.rb +++ b/lib/docs/scrapers/tensorflow.rb @@ -56,10 +56,8 @@ module Docs /\Aextend/] end - def get_latest_version(options, &block) - get_latest_github_release('tensorflow', 'tensorflow', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('tensorflow', 'tensorflow', opts)['tag_name'][1..-1] end private diff --git a/lib/docs/scrapers/terraform.rb b/lib/docs/scrapers/terraform.rb index 73d47e86..0965ad06 100644 --- a/lib/docs/scrapers/terraform.rb +++ b/lib/docs/scrapers/terraform.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MPL 2.0 License. HTML - def get_latest_version(options, &block) - get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', options) do |contents| - block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', opts) + contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] end end end diff --git a/lib/docs/scrapers/twig.rb b/lib/docs/scrapers/twig.rb index 58c7e2ef..e52ff6bd 100755 --- a/lib/docs/scrapers/twig.rb +++ b/lib/docs/scrapers/twig.rb @@ -29,10 +29,9 @@ module Docs self.base_url = 'https://twig.symfony.com/doc/1.x/' end - def get_latest_version(options, &block) - get_github_tags('twigphp', 'Twig', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('twigphp', 'Twig', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb index 9e62d6a9..63d1c9af 100644 --- a/lib/docs/scrapers/typescript.rb +++ b/lib/docs/scrapers/typescript.rb @@ -25,10 +25,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('Microsoft', 'TypeScript', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('Microsoft', 'TypeScript', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/underscore.rb b/lib/docs/scrapers/underscore.rb index 127dd5d3..a09b4acb 100644 --- a/lib/docs/scrapers/underscore.rb +++ b/lib/docs/scrapers/underscore.rb @@ -21,10 +21,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://underscorejs.org/', options) do |doc| - block.call doc.at_css('.version').content[1...-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://underscorejs.org/', opts) + doc.at_css('.version').content[1...-1] end end end diff --git a/lib/docs/scrapers/vagrant.rb b/lib/docs/scrapers/vagrant.rb index 0e89dad1..7400fc8c 100644 --- a/lib/docs/scrapers/vagrant.rb +++ b/lib/docs/scrapers/vagrant.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MPL 2.0 License. HTML - def get_latest_version(options, &block) - get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', options) do |contents| - block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', opts) + contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] end end end diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb index cbb6b15e..db48b898 100644 --- a/lib/docs/scrapers/vue.rb +++ b/lib/docs/scrapers/vue.rb @@ -33,10 +33,8 @@ module Docs self.initial_paths = %w(/api/index.html) end - def get_latest_version(options, &block) - get_latest_github_release('vuejs', 'vue', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('vuejs', 'vue', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/vulkan.rb b/lib/docs/scrapers/vulkan.rb index 111e405f..da5d696d 100644 --- a/lib/docs/scrapers/vulkan.rb +++ b/lib/docs/scrapers/vulkan.rb @@ -21,10 +21,9 @@ module Docs Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc. HTML - def get_latest_version(options, &block) - get_github_tags('KhronosGroup', 'Vulkan-Docs', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('KhronosGroup', 'Vulkan-Docs', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/webpack.rb b/lib/docs/scrapers/webpack.rb index 9d7303ed..7af2cf11 100644 --- a/lib/docs/scrapers/webpack.rb +++ b/lib/docs/scrapers/webpack.rb @@ -69,8 +69,8 @@ module Docs HTML end - def get_latest_version(options, &block) - get_npm_version('webpack', options, &block) + def get_latest_version(opts) + get_npm_version('webpack', opts) end end end diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb index 26f7c0cc..d63e7121 100644 --- a/lib/docs/scrapers/yarn.rb +++ b/lib/docs/scrapers/yarn.rb @@ -21,10 +21,8 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - get_latest_github_release('yarnpkg', 'yarn', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('yarnpkg', 'yarn', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/yii.rb b/lib/docs/scrapers/yii.rb index 0d087abb..b05c5bf7 100755 --- a/lib/docs/scrapers/yii.rb +++ b/lib/docs/scrapers/yii.rb @@ -35,10 +35,8 @@ module Docs options[:container] = '.grid_9' end - def get_latest_version(options, &block) - get_latest_github_release('yiisoft', 'yii2', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('yiisoft', 'yii2', opts)['tag_name'] end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index e684cdbc..b7e251fb 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -62,17 +62,15 @@ class UpdatesCLI < Thor logger.debug("Checking #{doc.name}") instance = doc.versions.first.new + scraper_version = instance.get_scraper_version(opts) + latest_version = instance.get_latest_version(opts) - instance.get_scraper_version(opts) do |scraper_version| - instance.get_latest_version(opts) do |latest_version| - return { - name: doc.name, - scraper_version: format_version(scraper_version), - latest_version: format_version(latest_version), - is_outdated: instance.is_outdated(scraper_version, latest_version) - } - end - end + { + name: doc.name, + scraper_version: format_version(scraper_version), + latest_version: format_version(latest_version), + is_outdated: instance.is_outdated(scraper_version, latest_version) + } rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") error_result(doc, '`get_latest_version` is not implemented') @@ -87,7 +85,7 @@ class UpdatesCLI < Thor # If the version is numeric and greater than or equal to 1e9 it's probably a timestamp return str if str.match(/^(\d)+$/).nil? or str.to_i < 1e9 - DateTime.strptime(str, '%s').strftime('%B %-d, %Y') + DateTime.strptime(str, '%s').strftime('%F') end def error_result(doc, reason) @@ -150,46 +148,45 @@ class UpdatesCLI < Thor logger.info('Uploading the results to a new GitHub issue') logger.info('Checking if the GitHub token belongs to the correct user') - github_get('/user') do |user| - # Only allow the DevDocs bot to upload reports - if user['login'] == UPLOAD_USER - issue = results_to_issue(outdated_results, up_to_date_results, failed_results) - - logger.info('Creating a new GitHub issue') - github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue| - search_params = { - q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", - sort: 'created', - order: 'desc' - } - - logger.info('Checking if the previous issue is still open') - github_get('/search/issues', search_params) do |matching_issues| - previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} - - if previous_issue.nil? - logger.info('No previous issue found') - log_upload_success(created_issue) - else - comment = "This report was superseded by ##{created_issue['number']}." - - logger.info('Commenting on the previous issue') - github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_| - if previous_issue['closed_at'].nil? - logger.info('Closing the previous issue') - github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_| - log_upload_success(created_issue) - end - else - logger.info('The previous issue has already been closed') - log_upload_success(created_issue) - end - end - end - end - end + user = github_get('/user') + + # Only allow the DevDocs bot to upload reports + unless user['login'] == UPLOAD_USER + logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + return + end + + logger.info('Creating a new GitHub issue') + + issue = results_to_issue(outdated_results, up_to_date_results, failed_results) + created_issue = github_post("/repos/#{UPLOAD_REPO}/issues", issue) + + logger.info('Checking if the previous issue is still open') + + search_params = { + q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", + sort: 'created', + order: 'desc' + } + + matching_issues = github_get('/search/issues', search_params) + previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} + + if previous_issue.nil? + logger.info('No previous issue found') + log_upload_success(created_issue) + else + logger.info('Commenting on the previous issue') + + comment = "This report was superseded by ##{created_issue['number']}." + github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) + if previous_issue['closed_at'].nil? + logger.info('Closing the previous issue') + github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) + log_upload_success(created_issue) else - logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + logger.info('The previous issue has already been closed') + log_upload_success(created_issue) end end end @@ -266,19 +263,19 @@ The #{outdated_results.length + up_to_date_results.length + failed_results.lengt # HTTP utilities # - def github_get(endpoint, params = {}, &block) - github_request(endpoint, {method: :get, params: params}, &block) + def github_get(endpoint, params = {}) + github_request(endpoint, {method: :get, params: params}) end - def github_post(endpoint, params, &block) - github_request(endpoint, {method: :post, body: params.to_json}, &block) + def github_post(endpoint, params) + github_request(endpoint, {method: :post, body: params.to_json}) end - def github_patch(endpoint, params, &block) - github_request(endpoint, {method: :patch, body: params.to_json}, &block) + def github_patch(endpoint, params) + github_request(endpoint, {method: :patch, body: params.to_json}) end - def github_request(endpoint, opts, &block) + def github_request(endpoint, opts) url = "https://api.github.com#{endpoint}" # GitHub token authentication @@ -292,16 +289,15 @@ The #{outdated_results.length + up_to_date_results.length + failed_results.lengt end logger.debug("Making a #{opts[:method]} request to #{url}") - - Docs::Request.run(url, opts) do |response| - # response.success? is false if the response code is 201 - # GitHub returns 201 Created after an issue is created - if response.success? || response.code == 201 - block.call JSON.parse(response.body) - else - logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") - block.call nil - end + response = Docs::Request.run(url, opts) + + # response.success? is false if the response code is 201 + # GitHub returns 201 Created after an issue is created + if response.success? || response.code == 201 + JSON.parse(response.body) + else + logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") + nil end end From 227fbfe70f5422e10ebba3c2d82bfa42c8d49335 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 03:26:23 +0100 Subject: [PATCH 10/17] Update issue message and issue repository --- lib/tasks/updates.thor | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index b7e251fb..66134581 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -1,11 +1,9 @@ class UpdatesCLI < Thor # The GitHub user that is allowed to upload reports - # TODO: Update this before creating a PR UPLOAD_USER = 'devdocs-bot' # The repository to create an issue in when uploading the results - # TODO: Update this before creating a PR - UPLOAD_REPO = 'jmerle/devdocs' + UPLOAD_REPO = 'freeCodeCamp/devdocs' def self.to_s 'Updates' @@ -207,7 +205,7 @@ class UpdatesCLI < Thor This is an automatically created issue which contains information about the version status of the documentations available on DevDocs. The results of this report can be used by maintainers when updating outdated documentations. -Maintainers can close this issue when all documentations are up-to-date. This issue is automatically closed when the next report is created.#{travis_str} +Maintainers can close this issue when all documentations are up-to-date. The issue is also automatically closed when the next report is created.#{travis_str} ## Results From 1b35b21dd6021371a3c5b27608d4582a9be188c2 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 04:05:14 +0100 Subject: [PATCH 11/17] Update contributing.md --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d1f848bd..d1f1f851 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -61,7 +61,7 @@ In addition to the [guidelines for contributing code](#contributing-code-and-fea Please don't submit a pull request updating the version number of a documentation, unless a change is required in the scraper and you've verified that it works. -To ask that an existing documentation be updated, please use the [Trello board](https://trello.com/c/2B0hmW7M/52-request-updates-here). +To ask that an existing documentation be updated, first check the last two [Documentation versions reports](https://github.com/freeCodeCamp/devdocs/issues?utf8=%E2%9C%93&q=Documentation+versions+report+is%3Aissue+author%3Adevdocs-bot+sort%3Aupdated-desc). Only create an issue if the documentation has been wrongly marked as up-to-date for at least 2 reports (a new report is automatically created every month). ## Coding conventions From 11eeadbc0a7a6b098cdf04775e4738b01cbabba5 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 23:15:40 +0100 Subject: [PATCH 12/17] Better error handling and added a 15s timeout on HTTP requests --- lib/docs/core/doc.rb | 7 ++++--- lib/tasks/updates.thor | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index 0b913ef6..4bdfcf15 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -223,13 +223,14 @@ module Docs end opts[:logger].debug("Fetching #{url}") - response = Request.run(url, { headers: headers }) + response = Request.run(url, { connecttimeout: 15, headers: headers }) if response.success? response.body else - opts[:logger].error("Couldn't fetch #{url} (response code #{response.code})") - nil + reason = response.timed_out? ? "Timed out while fetching #{url}" : "Couldn't fetch #{url} (response code #{response.code})" + opts[:logger].error(reason) + raise reason end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index 66134581..20ef20b2 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -72,9 +72,9 @@ class UpdatesCLI < Thor rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") error_result(doc, '`get_latest_version` is not implemented') - rescue - logger.error("Error while checking #{doc.name}") - raise + rescue => error + logger.error("Error while checking #{doc.name}\n#{error.full_message.strip}") + error_result(doc, error.message.gsub(/'/, '`')) end def format_version(version) From 3cb97f71b32ad622b513655372df291b5216ce93 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Mon, 11 Mar 2019 10:47:32 +0100 Subject: [PATCH 13/17] fetching -> connecting to --- lib/docs/core/doc.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index 4bdfcf15..b50957c5 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -228,7 +228,7 @@ module Docs if response.success? response.body else - reason = response.timed_out? ? "Timed out while fetching #{url}" : "Couldn't fetch #{url} (response code #{response.code})" + reason = response.timed_out? ? "Timed out while connecting to #{url}" : "Couldn't fetch #{url} (response code #{response.code})" opts[:logger].error(reason) raise reason end From 4ce7dbcfd58873857fc4c613d4c6b4bbb9d90f12 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Tue, 11 Jun 2019 23:01:37 +0200 Subject: [PATCH 14/17] Remove hardcoded username and fix broken methods --- .github/CONTRIBUTING.md | 2 +- lib/docs/scrapers/async.rb | 2 +- lib/docs/scrapers/drupal.rb | 9 ++------- lib/docs/scrapers/perl.rb | 5 +++-- lib/docs/scrapers/phalcon.rb | 2 +- lib/docs/scrapers/sass.rb | 2 +- lib/docs/scrapers/statsmodels.rb | 2 +- lib/tasks/updates.thor | 2 +- 8 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 06558d36..d0fccff5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -61,7 +61,7 @@ In addition to the [guidelines for contributing code](#contributing-code-and-fea Please don't submit a pull request updating the version number of a documentation, unless a change is required in the scraper and you've verified that it works. -To ask that an existing documentation be updated, first check the last two [Documentation versions reports](https://github.com/freeCodeCamp/devdocs/issues?utf8=%E2%9C%93&q=Documentation+versions+report+is%3Aissue+author%3Adevdocs-bot+sort%3Aupdated-desc). Only create an issue if the documentation has been wrongly marked as up-to-date for at least 2 reports (a new report is automatically created every month). +To ask that an existing documentation be updated, first check the last two [Documentation versions reports](https://github.com/freeCodeCamp/devdocs/issues?utf8=%E2%9C%93&q=Documentation+versions+report+is%3Aissue+author%3Adevdocs-bot+sort%3Aupdated-desc). Only create an issue if the documentation has been wrongly marked as up-to-date. ## Coding conventions diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 67498eed..61615b54 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -19,7 +19,7 @@ module Docs HTML def get_latest_version(opts) - doc = fetch_doc('https://caolan.github.io/async/', opts) + doc = fetch_doc('https://caolan.github.io/async/v3/', opts) doc.at_css('#version-dropdown > a').content.strip[1..-1] end end diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb index f29b585b..3798caec 100644 --- a/lib/docs/scrapers/drupal.rb +++ b/lib/docs/scrapers/drupal.rb @@ -100,13 +100,8 @@ module Docs end def get_latest_version(opts) - doc = fetch_doc('http://cgit.drupalcode.org/drupal', opts) - - version = doc.at_css('td.form > form > select > option[selected]').content - version = version.scan(/([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - - version + json = fetch_json('https://packagist.org/packages/drupal/drupal.json', opts) + json['package']['versions'].keys.find {|version| !version.end_with?('-dev')} end end end diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb index 7e5ed8f8..ebf0a653 100644 --- a/lib/docs/scrapers/perl.rb +++ b/lib/docs/scrapers/perl.rb @@ -45,8 +45,9 @@ module Docs end def get_latest_version(opts) - body = fetch('https://perldoc.perl.org/static/perlversion.js', opts) - body.scan(/>Perl ([0-9.]+)/)[0][0] + doc = fetch_doc('https://perldoc.perl.org/', opts) + header = doc.at_css('h2.h1').content + header.scan(/Perl ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/phalcon.rb b/lib/docs/scrapers/phalcon.rb index dd476329..c6ca63f2 100644 --- a/lib/docs/scrapers/phalcon.rb +++ b/lib/docs/scrapers/phalcon.rb @@ -32,7 +32,7 @@ module Docs def get_latest_version(opts) doc = fetch_doc('https://docs.phalconphp.com/', opts) - doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') + doc.at_css('.header__lang.expand > div > ul > li > a').content end end end diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb index 228a5337..c81753d2 100644 --- a/lib/docs/scrapers/sass.rb +++ b/lib/docs/scrapers/sass.rb @@ -25,7 +25,7 @@ module Docs HTML def get_latest_version(opts) - get_github_file_contents('sass', 'sass', 'VERSION', opts).strip + get_latest_github_release('sass', 'libsass', opts)['tag_name'] end end end diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb index db2eacb2..3e8a357c 100644 --- a/lib/docs/scrapers/statsmodels.rb +++ b/lib/docs/scrapers/statsmodels.rb @@ -23,7 +23,7 @@ module Docs def get_latest_version(opts) doc = fetch_doc('http://www.statsmodels.org/stable/', opts) - doc.at_css('.sphinxsidebarwrapper h3 + p > b').content + doc.at_css('.sphinxsidebarwrapper h3 + p > b').content[1..-1] end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index 20ef20b2..c715f752 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -139,7 +139,7 @@ class UpdatesCLI < Thor def upload_results(outdated_results, up_to_date_results, failed_results) # We can't create issues without a GitHub token unless options.key?(:github_token) - logger.error('Please specify a GitHub token with the public_repo permission for devdocs-bot with the --github-token parameter') + logger.error("Please specify a GitHub token with the public_repo permission for #{UPLOAD_USER} with the --github-token parameter") return end From 7084d9f082e589e651f5296896c5d76c55ad0c75 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Thu, 20 Jun 2019 21:40:15 +0200 Subject: [PATCH 15/17] Improve get_latest_github_release --- docs/scraper-reference.md | 2 +- lib/docs/core/doc.rb | 4 +++- lib/docs/scrapers/homebrew.rb | 2 +- lib/docs/scrapers/jasmine.rb | 2 +- lib/docs/scrapers/jsdoc.rb | 2 +- lib/docs/scrapers/julia.rb | 2 +- lib/docs/scrapers/knockout.rb | 2 +- lib/docs/scrapers/kotlin.rb | 2 +- lib/docs/scrapers/laravel.rb | 2 +- lib/docs/scrapers/matplotlib.rb | 2 +- lib/docs/scrapers/nokogiri2.rb | 2 +- lib/docs/scrapers/npm.rb | 2 +- lib/docs/scrapers/numpy.rb | 2 +- lib/docs/scrapers/opentsdb.rb | 2 +- lib/docs/scrapers/phaser.rb | 2 +- lib/docs/scrapers/pygame.rb | 2 +- lib/docs/scrapers/rdoc/rails.rb | 2 +- lib/docs/scrapers/rethinkdb.rb | 2 +- lib/docs/scrapers/sass.rb | 2 +- lib/docs/scrapers/symfony.rb | 2 +- lib/docs/scrapers/tensorflow.rb | 2 +- lib/docs/scrapers/typescript.rb | 2 +- lib/docs/scrapers/vue.rb | 2 +- lib/docs/scrapers/yarn.rb | 2 +- lib/docs/scrapers/yii.rb | 2 +- 25 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index de7d3f15..4da0d2e5 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -212,7 +212,7 @@ To make life easier, there are a few utility methods that you can use in `get_la Example: [lib/docs/scrapers/bower.rb](../lib/docs/scrapers/bower.rb) * `get_latest_github_release(owner, repo, opts)` - Returns the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). + Returns the tag name of the latest GitHub release of the given repository. If the tag name is preceeded by a "v", the "v" will be removed. Example: [lib/docs/scrapers/jsdoc.rb](../lib/docs/scrapers/jsdoc.rb) * `get_github_tags(owner, repo, opts)` diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index b50957c5..da21daf8 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -249,7 +249,9 @@ module Docs end def get_latest_github_release(owner, repo, opts) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", opts) + release = fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", opts) + tag_name = release['tag_name'] + tag_name.start_with?('v') ? tag_name[1..-1] : tag_name end def get_github_tags(owner, repo, opts) diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index c5647709..9dd1581a 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -21,7 +21,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('Homebrew', 'brew', opts)['name'] + get_latest_github_release('Homebrew', 'brew', opts) end end end diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb index 14c51869..b1971ecd 100644 --- a/lib/docs/scrapers/jasmine.rb +++ b/lib/docs/scrapers/jasmine.rb @@ -19,7 +19,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('jasmine', 'jasmine', opts)['name'] + get_latest_github_release('jasmine', 'jasmine', opts) end end end diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb index d88d46b6..df27e578 100644 --- a/lib/docs/scrapers/jsdoc.rb +++ b/lib/docs/scrapers/jsdoc.rb @@ -23,7 +23,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('jsdoc3', 'jsdoc', opts)['tag_name'] + get_latest_github_release('jsdoc3', 'jsdoc', opts) end end end diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb index d152f05d..c9c96da6 100644 --- a/lib/docs/scrapers/julia.rb +++ b/lib/docs/scrapers/julia.rb @@ -51,7 +51,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('JuliaLang', 'julia', opts)['tag_name'][1..-1] + get_latest_github_release('JuliaLang', 'julia', opts) end end end diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb index 6556bca8..efad86f0 100644 --- a/lib/docs/scrapers/knockout.rb +++ b/lib/docs/scrapers/knockout.rb @@ -35,7 +35,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('knockout', 'knockout', opts)['tag_name'][1..-1] + get_latest_github_release('knockout', 'knockout', opts) end end end diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb index 5f508ae7..5055b65e 100644 --- a/lib/docs/scrapers/kotlin.rb +++ b/lib/docs/scrapers/kotlin.rb @@ -30,7 +30,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('JetBrains', 'kotlin', opts)['tag_name'][1..-1] + get_latest_github_release('JetBrains', 'kotlin', opts) end end end diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index 4fc17368..e45b0bed 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -135,7 +135,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('laravel', 'laravel', opts)['tag_name'][1..-1] + get_latest_github_release('laravel', 'laravel', opts) end end end diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb index 4a882270..eeecea71 100644 --- a/lib/docs/scrapers/matplotlib.rb +++ b/lib/docs/scrapers/matplotlib.rb @@ -66,7 +66,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('matplotlib', 'matplotlib', opts)['tag_name'][1..-1] + get_latest_github_release('matplotlib', 'matplotlib', opts) end end end diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb index 9da5daf4..7c28ca92 100644 --- a/lib/docs/scrapers/nokogiri2.rb +++ b/lib/docs/scrapers/nokogiri2.rb @@ -21,7 +21,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('sparklemotion', 'nokogiri', opts)['tag_name'][1..-1] + get_latest_github_release('sparklemotion', 'nokogiri', opts) end end end diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb index e18531ab..3f868a3c 100644 --- a/lib/docs/scrapers/npm.rb +++ b/lib/docs/scrapers/npm.rb @@ -31,7 +31,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('npm', 'cli', opts)['tag_name'][1..-1] + get_latest_github_release('npm', 'cli', opts) end end end diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb index 636fcf4a..84de6cab 100644 --- a/lib/docs/scrapers/numpy.rb +++ b/lib/docs/scrapers/numpy.rb @@ -51,7 +51,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('numpy', 'numpy', opts)['tag_name'][1..-1] + get_latest_github_release('numpy', 'numpy', opts) end end end diff --git a/lib/docs/scrapers/opentsdb.rb b/lib/docs/scrapers/opentsdb.rb index 1de40478..6eec407c 100644 --- a/lib/docs/scrapers/opentsdb.rb +++ b/lib/docs/scrapers/opentsdb.rb @@ -20,7 +20,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('OpenTSDB', 'opentsdb', opts)['tag_name'][1..-1] + get_latest_github_release('OpenTSDB', 'opentsdb', opts) end end end diff --git a/lib/docs/scrapers/phaser.rb b/lib/docs/scrapers/phaser.rb index 1939b1d0..bd5f411c 100644 --- a/lib/docs/scrapers/phaser.rb +++ b/lib/docs/scrapers/phaser.rb @@ -27,7 +27,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('photonstorm', 'phaser', opts)['tag_name'][1..-1] + get_latest_github_release('photonstorm', 'phaser', opts) end end end diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb index d5a5581d..94c3d508 100644 --- a/lib/docs/scrapers/pygame.rb +++ b/lib/docs/scrapers/pygame.rb @@ -19,7 +19,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('pygame', 'pygame', opts)['tag_name'] + get_latest_github_release('pygame', 'pygame', opts) end end end diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb index 771c2f64..0dec42a9 100644 --- a/lib/docs/scrapers/rdoc/rails.rb +++ b/lib/docs/scrapers/rdoc/rails.rb @@ -95,7 +95,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('rails', 'rails', opts)['name'] + get_latest_github_release('rails', 'rails', opts) end end end diff --git a/lib/docs/scrapers/rethinkdb.rb b/lib/docs/scrapers/rethinkdb.rb index 3a6b87cf..35d8d334 100644 --- a/lib/docs/scrapers/rethinkdb.rb +++ b/lib/docs/scrapers/rethinkdb.rb @@ -59,7 +59,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('rethinkdb', 'rethinkdb', opts)['tag_name'][1..-1] + get_latest_github_release('rethinkdb', 'rethinkdb', opts) end private diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb index c81753d2..c9774cdf 100644 --- a/lib/docs/scrapers/sass.rb +++ b/lib/docs/scrapers/sass.rb @@ -25,7 +25,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('sass', 'libsass', opts)['tag_name'] + get_latest_github_release('sass', 'libsass', opts) end end end diff --git a/lib/docs/scrapers/symfony.rb b/lib/docs/scrapers/symfony.rb index 439456f0..4bdde054 100644 --- a/lib/docs/scrapers/symfony.rb +++ b/lib/docs/scrapers/symfony.rb @@ -72,7 +72,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('symfony', 'symfony', opts)['tag_name'][1..-1] + get_latest_github_release('symfony', 'symfony', opts) end end end diff --git a/lib/docs/scrapers/tensorflow.rb b/lib/docs/scrapers/tensorflow.rb index 8ac1ca65..9638b516 100644 --- a/lib/docs/scrapers/tensorflow.rb +++ b/lib/docs/scrapers/tensorflow.rb @@ -57,7 +57,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('tensorflow', 'tensorflow', opts)['tag_name'][1..-1] + get_latest_github_release('tensorflow', 'tensorflow', opts) end private diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb index 63d1c9af..777f5dd6 100644 --- a/lib/docs/scrapers/typescript.rb +++ b/lib/docs/scrapers/typescript.rb @@ -26,7 +26,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('Microsoft', 'TypeScript', opts)['tag_name'][1..-1] + get_latest_github_release('Microsoft', 'TypeScript', opts) end end end diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb index db48b898..333ade1d 100644 --- a/lib/docs/scrapers/vue.rb +++ b/lib/docs/scrapers/vue.rb @@ -34,7 +34,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('vuejs', 'vue', opts)['tag_name'][1..-1] + get_latest_github_release('vuejs', 'vue', opts) end end end diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb index d63e7121..c45695af 100644 --- a/lib/docs/scrapers/yarn.rb +++ b/lib/docs/scrapers/yarn.rb @@ -22,7 +22,7 @@ module Docs HTML def get_latest_version(opts) - get_latest_github_release('yarnpkg', 'yarn', opts)['tag_name'][1..-1] + get_latest_github_release('yarnpkg', 'yarn', opts) end end end diff --git a/lib/docs/scrapers/yii.rb b/lib/docs/scrapers/yii.rb index b05c5bf7..eaabdbf0 100755 --- a/lib/docs/scrapers/yii.rb +++ b/lib/docs/scrapers/yii.rb @@ -36,7 +36,7 @@ module Docs end def get_latest_version(opts) - get_latest_github_release('yiisoft', 'yii2', opts)['tag_name'] + get_latest_github_release('yiisoft', 'yii2', opts) end end end From 0529f8087055ec87a9158063004357ebd2aaad2c Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Thu, 20 Jun 2019 21:42:19 +0200 Subject: [PATCH 16/17] Fix typo --- docs/scraper-reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scraper-reference.md b/docs/scraper-reference.md index 4da0d2e5..c2872388 100644 --- a/docs/scraper-reference.md +++ b/docs/scraper-reference.md @@ -212,7 +212,7 @@ To make life easier, there are a few utility methods that you can use in `get_la Example: [lib/docs/scrapers/bower.rb](../lib/docs/scrapers/bower.rb) * `get_latest_github_release(owner, repo, opts)` - Returns the tag name of the latest GitHub release of the given repository. If the tag name is preceeded by a "v", the "v" will be removed. + Returns the tag name of the latest GitHub release of the given repository. If the tag name is preceded by a "v", the "v" will be removed. Example: [lib/docs/scrapers/jsdoc.rb](../lib/docs/scrapers/jsdoc.rb) * `get_github_tags(owner, repo, opts)` From 5ac8d55ed6fc49a0b06b08dfa24dccd19d45365d Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Fri, 19 Jul 2019 21:44:18 +0200 Subject: [PATCH 17/17] Fix Go get_latest_version --- lib/docs/scrapers/go.rb | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb index 8997b7b1..27b46652 100644 --- a/lib/docs/scrapers/go.rb +++ b/lib/docs/scrapers/go.rb @@ -25,13 +25,8 @@ module Docs HTML def get_latest_version(opts) - doc = fetch_doc('https://golang.org/pkg/', opts) - - footer = doc.at_css('#footer').content - version = footer.scan(/go([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - - version + doc = fetch_doc('https://golang.org/project/', opts) + doc.at_css('#page ul > li > a').text[3..-1] end private