From 41e0a138a6533ed39f88d242db051ce33287e505 Mon Sep 17 00:00:00 2001 From: Jasper van Merle Date: Sun, 10 Mar 2019 03:02:24 +0100 Subject: [PATCH] Refactoring and cleaning up --- docs/Scraper-Reference.md | 32 +++--- lib/docs/core/doc.rb | 71 ++++++------ lib/docs/scrapers/angular.rb | 4 +- lib/docs/scrapers/angularjs.rb | 4 +- lib/docs/scrapers/ansible.rb | 7 +- lib/docs/scrapers/apache.rb | 7 +- lib/docs/scrapers/apache_pig.rb | 9 +- lib/docs/scrapers/async.rb | 8 +- lib/docs/scrapers/babel.rb | 7 +- lib/docs/scrapers/backbone.rb | 8 +- lib/docs/scrapers/bash.rb | 8 +- lib/docs/scrapers/bluebird.rb | 4 +- lib/docs/scrapers/bootstrap.rb | 7 +- lib/docs/scrapers/bottle.rb | 9 +- lib/docs/scrapers/bower.rb | 4 +- lib/docs/scrapers/c.rb | 11 +- lib/docs/scrapers/cakephp.rb | 7 +- lib/docs/scrapers/chai.rb | 4 +- lib/docs/scrapers/chef.rb | 7 +- lib/docs/scrapers/clojure.rb | 7 +- lib/docs/scrapers/cmake.rb | 9 +- lib/docs/scrapers/codeception.rb | 7 +- lib/docs/scrapers/codeceptjs.rb | 4 +- lib/docs/scrapers/codeigniter.rb | 9 +- lib/docs/scrapers/coffeescript.rb | 4 +- lib/docs/scrapers/cordova.rb | 15 +-- lib/docs/scrapers/cpp.rb | 11 +- lib/docs/scrapers/crystal.rb | 7 +- lib/docs/scrapers/d.rb | 7 +- lib/docs/scrapers/d3.rb | 4 +- lib/docs/scrapers/dart.rb | 9 +- lib/docs/scrapers/django.rb | 7 +- lib/docs/scrapers/docker.rb | 9 +- lib/docs/scrapers/dojo.rb | 7 +- lib/docs/scrapers/drupal.rb | 15 +-- lib/docs/scrapers/electron.rb | 7 +- lib/docs/scrapers/elixir.rb | 7 +- lib/docs/scrapers/ember.rb | 7 +- lib/docs/scrapers/erlang.rb | 7 +- lib/docs/scrapers/eslint.rb | 4 +- lib/docs/scrapers/express.rb | 4 +- lib/docs/scrapers/falcon.rb | 7 +- lib/docs/scrapers/fish.rb | 7 +- lib/docs/scrapers/flow.rb | 4 +- lib/docs/scrapers/git.rb | 7 +- lib/docs/scrapers/gnu/gcc.rb | 9 +- lib/docs/scrapers/gnu/gnu_fortran.rb | 9 +- lib/docs/scrapers/go.rb | 15 +-- lib/docs/scrapers/godot.rb | 7 +- lib/docs/scrapers/graphite.rb | 7 +- lib/docs/scrapers/grunt.rb | 4 +- lib/docs/scrapers/handlebars.rb | 4 +- lib/docs/scrapers/haskell.rb | 11 +- lib/docs/scrapers/haxe.rb | 9 +- lib/docs/scrapers/homebrew.rb | 6 +- lib/docs/scrapers/immutable.rb | 4 +- lib/docs/scrapers/influxdata.rb | 9 +- lib/docs/scrapers/jasmine.rb | 6 +- lib/docs/scrapers/jekyll.rb | 7 +- lib/docs/scrapers/jest.rb | 7 +- lib/docs/scrapers/jquery/jquery_core.rb | 4 +- lib/docs/scrapers/jquery/jquery_mobile.rb | 8 +- lib/docs/scrapers/jquery/jquery_ui.rb | 4 +- lib/docs/scrapers/jsdoc.rb | 6 +- lib/docs/scrapers/julia.rb | 6 +- lib/docs/scrapers/knockout.rb | 6 +- lib/docs/scrapers/koa.rb | 4 +- lib/docs/scrapers/kotlin.rb | 6 +- lib/docs/scrapers/laravel.rb | 6 +- lib/docs/scrapers/leaflet.rb | 9 +- lib/docs/scrapers/less.rb | 9 +- lib/docs/scrapers/liquid.rb | 7 +- lib/docs/scrapers/lodash.rb | 7 +- lib/docs/scrapers/love.rb | 7 +- lib/docs/scrapers/lua.rb | 7 +- lib/docs/scrapers/marionette.rb | 4 +- lib/docs/scrapers/markdown.rb | 4 +- lib/docs/scrapers/matplotlib.rb | 6 +- lib/docs/scrapers/mdn/mdn.rb | 7 +- lib/docs/scrapers/meteor.rb | 7 +- lib/docs/scrapers/mocha.rb | 4 +- lib/docs/scrapers/modernizr.rb | 4 +- lib/docs/scrapers/moment.rb | 7 +- lib/docs/scrapers/mongoose.rb | 9 +- lib/docs/scrapers/nginx.rb | 9 +- lib/docs/scrapers/nginx_lua_module.rb | 9 +- lib/docs/scrapers/nim.rb | 7 +- lib/docs/scrapers/node.rb | 7 +- lib/docs/scrapers/nokogiri2.rb | 6 +- lib/docs/scrapers/npm.rb | 6 +- lib/docs/scrapers/numpy.rb | 6 +- lib/docs/scrapers/openjdk.rb | 17 ++- lib/docs/scrapers/opentsdb.rb | 6 +- lib/docs/scrapers/padrino.rb | 6 +- lib/docs/scrapers/pandas.rb | 9 +- lib/docs/scrapers/perl.rb | 7 +- lib/docs/scrapers/phalcon.rb | 7 +- lib/docs/scrapers/phaser.rb | 6 +- lib/docs/scrapers/phoenix.rb | 7 +- lib/docs/scrapers/php.rb | 9 +- lib/docs/scrapers/phpunit.rb | 9 +- lib/docs/scrapers/postgresql.rb | 9 +- lib/docs/scrapers/pug.rb | 4 +- lib/docs/scrapers/puppeteer.rb | 7 +- lib/docs/scrapers/pygame.rb | 6 +- lib/docs/scrapers/python.rb | 7 +- lib/docs/scrapers/q.rb | 4 +- lib/docs/scrapers/qt.rb | 8 +- lib/docs/scrapers/ramda.rb | 7 +- lib/docs/scrapers/rdoc/minitest.rb | 7 +- lib/docs/scrapers/rdoc/rails.rb | 6 +- lib/docs/scrapers/rdoc/ruby.rb | 15 ++- lib/docs/scrapers/react.rb | 7 +- lib/docs/scrapers/react_native.rb | 7 +- lib/docs/scrapers/redis.rb | 9 +- lib/docs/scrapers/redux.rb | 4 +- lib/docs/scrapers/relay.rb | 7 +- lib/docs/scrapers/requirejs.rb | 4 +- lib/docs/scrapers/rethinkdb.rb | 6 +- lib/docs/scrapers/rust.rb | 9 +- lib/docs/scrapers/sass.rb | 6 +- lib/docs/scrapers/scikit_image.rb | 7 +- lib/docs/scrapers/scikit_learn.rb | 8 +- lib/docs/scrapers/sinon.rb | 7 +- lib/docs/scrapers/socketio.rb | 4 +- lib/docs/scrapers/sqlite.rb | 7 +- lib/docs/scrapers/statsmodels.rb | 7 +- lib/docs/scrapers/support_tables.rb | 9 +- lib/docs/scrapers/symfony.rb | 6 +- lib/docs/scrapers/tcl_tk.rb | 8 +- lib/docs/scrapers/tensorflow.rb | 6 +- lib/docs/scrapers/terraform.rb | 7 +- lib/docs/scrapers/twig.rb | 7 +- lib/docs/scrapers/typescript.rb | 6 +- lib/docs/scrapers/underscore.rb | 7 +- lib/docs/scrapers/vagrant.rb | 7 +- lib/docs/scrapers/vue.rb | 6 +- lib/docs/scrapers/vulkan.rb | 7 +- lib/docs/scrapers/webpack.rb | 4 +- lib/docs/scrapers/yarn.rb | 6 +- lib/docs/scrapers/yii.rb | 6 +- lib/tasks/updates.thor | 130 +++++++++++----------- 142 files changed, 530 insertions(+), 676 deletions(-) diff --git a/docs/Scraper-Reference.md b/docs/Scraper-Reference.md index a6736a3e..de7d3f15 100644 --- a/docs/Scraper-Reference.md +++ b/docs/Scraper-Reference.md @@ -187,39 +187,41 @@ More information about how filters work is available on the [Filter Reference](. ## Keeping scrapers up-to-date -In order to keep scrapers up-to-date the `get_latest_version(options, &block)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. +In order to keep scrapers up-to-date the `get_latest_version(opts)` method should be overridden. If `self.release` is defined, this should return the latest version of the documentation. If `self.release` is not defined, it should return the Epoch time when the documentation was last modified. If the documentation will never change, simply return `1.0.0`. The result of this method is periodically reported in a "Documentation versions report" issue which helps maintainers keep track of outdated documentations. To make life easier, there are a few utility methods that you can use in `get_latest_version`: -* `fetch(url, options, &block)` +* `fetch(url, opts)` - Makes a GET request to the url and calls `&block` with the body. + Makes a GET request to the url and returns the response body. Example: [lib/docs/scrapers/bash.rb](../lib/docs/scrapers/bash.rb) -* `fetch_doc(url, options, &block)` +* `fetch_doc(url, opts)` - Makes a GET request to the url and calls `&block` with the HTML body converted to a Nokogiri document. + Makes a GET request to the url and returns the HTML body converted to a Nokogiri document. Example: [lib/docs/scrapers/git.rb](../lib/docs/scrapers/git.rb) -* `fetch_json(url, options, &block)` +* `fetch_json(url, opts)` - Makes a GET request to the url and calls `&block` with the JSON body converted to a dictionary. -* `get_npm_version(package, options, &block)` + Makes a GET request to the url and returns the JSON body converted to a dictionary. - Calls `&block` with the latest version of the given npm package. + Example: [lib/docs/scrapers/mdn/mdn.rb](../lib/docs/scrapers/mdn/mdn.rb) +* `get_npm_version(package, opts)` + + Returns the latest version of the given npm package. Example: [lib/docs/scrapers/bower.rb](../lib/docs/scrapers/bower.rb) -* `get_latest_github_release(owner, repo, options, &block)` +* `get_latest_github_release(owner, repo, opts)` - Calls `&block` with the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). + Returns the latest GitHub release of the given repository ([format](https://developer.github.com/v3/repos/releases/#get-the-latest-release)). Example: [lib/docs/scrapers/jsdoc.rb](../lib/docs/scrapers/jsdoc.rb) -* `get_github_tags(owner, repo, options, &block)` +* `get_github_tags(owner, repo, opts)` - Calls `&block` with the list of tags on the given repository ([format](https://developer.github.com/v3/repos/#list-tags)). + Returns the list of tags on the given repository ([format](https://developer.github.com/v3/repos/#list-tags)). Example: [lib/docs/scrapers/liquid.rb](../lib/docs/scrapers/liquid.rb) -* `get_github_file_contents(owner, repo, path, options, &block)` +* `get_github_file_contents(owner, repo, path, opts)` - Calls `&block` with the contents of the requested file in the default branch of the given repository. + Returns the contents of the requested file in the default branch of the given repository. Example: [lib/docs/scrapers/minitest.rb](../lib/docs/scrapers/minitest.rb) diff --git a/lib/docs/core/doc.rb b/lib/docs/core/doc.rb index 062ac9e7..0b913ef6 100644 --- a/lib/docs/core/doc.rb +++ b/lib/docs/core/doc.rb @@ -164,16 +164,15 @@ module Docs raise NotImplementedError end - def get_scraper_version(opts, &block) + def get_scraper_version(opts) if self.class.method_defined?(:options) and !options[:release].nil? - block.call options[:release] + options[:release] else # If options[:release] does not exist, we return the Epoch timestamp of when the doc was last modified in DevDocs production - fetch_json('https://devdocs.io/docs.json', opts) do |json| - items = json.select {|item| item['name'] == self.class.name} - items = items.map {|item| item['mtime']} - block.call items.max - end + json = fetch_json('https://devdocs.io/docs.json', opts) + items = json.select {|item| item['name'] == self.class.name} + items = items.map {|item| item['mtime']} + items.max end end @@ -181,7 +180,7 @@ module Docs # If options[:release] is defined, it should be in the same format # If options[:release] is not defined, it should return the Epoch timestamp of when the documentation was last updated # If the docs will never change, simply return '1.0.0' - def get_latest_version(options, &block) + def get_latest_version(opts) raise NotImplementedError end @@ -216,55 +215,49 @@ module Docs # Utility methods for get_latest_version # - def fetch(url, options, &block) + def fetch(url, opts) headers = {} - if options.key?(:github_token) and url.start_with?('https://api.github.com/') - headers['Authorization'] = "token #{options[:github_token]}" + if opts.key?(:github_token) and url.start_with?('https://api.github.com/') + headers['Authorization'] = "token #{opts[:github_token]}" end - options[:logger].debug("Fetching #{url}") + opts[:logger].debug("Fetching #{url}") + response = Request.run(url, { headers: headers }) - Request.run(url, { headers: headers }) do |response| - if response.success? - block.call response.body - else - options[:logger].error("Couldn't fetch #{url} (response code #{response.code})") - block.call nil - end + if response.success? + response.body + else + opts[:logger].error("Couldn't fetch #{url} (response code #{response.code})") + nil end end - def fetch_doc(url, options, &block) - fetch(url, options) do |body| - block.call Nokogiri::HTML.parse(body, nil, 'UTF-8') - end + def fetch_doc(url, opts) + body = fetch(url, opts) + Nokogiri::HTML.parse(body, nil, 'UTF-8') end - def fetch_json(url, options, &block) - fetch(url, options) do |body| - block.call JSON.parse(body) - end + def fetch_json(url, opts) + JSON.parse fetch(url, opts) end - def get_npm_version(package, options, &block) - fetch_json("https://registry.npmjs.com/#{package}", options) do |json| - block.call json['dist-tags']['latest'] - end + def get_npm_version(package, opts) + json = fetch_json("https://registry.npmjs.com/#{package}", opts) + json['dist-tags']['latest'] end - def get_latest_github_release(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", options, &block) + def get_latest_github_release(owner, repo, opts) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/releases/latest", opts) end - def get_github_tags(owner, repo, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", options, &block) + def get_github_tags(owner, repo, opts) + fetch_json("https://api.github.com/repos/#{owner}/#{repo}/tags", opts) end - def get_github_file_contents(owner, repo, path, options, &block) - fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", options) do |json| - block.call(Base64.decode64(json['content'])) - end + def get_github_file_contents(owner, repo, path, opts) + json = fetch_json("https://api.github.com/repos/#{owner}/#{repo}/contents/#{path}", opts) + Base64.decode64(json['content']) end end end diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index 059b0e8e..3365ec67 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -155,8 +155,8 @@ module Docs end end - def get_latest_version(options, &block) - get_npm_version('@angular/core', options, &block) + def get_latest_version(opts) + get_npm_version('@angular/core', opts) end private diff --git a/lib/docs/scrapers/angularjs.rb b/lib/docs/scrapers/angularjs.rb index b6e18325..9d663e35 100644 --- a/lib/docs/scrapers/angularjs.rb +++ b/lib/docs/scrapers/angularjs.rb @@ -70,8 +70,8 @@ module Docs self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end - def get_latest_version(options, &block) - get_npm_version('angular', options, &block) + def get_latest_version(opts) + get_npm_version('angular', opts) end end end diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb index 293f74a7..b2363d4d 100644 --- a/lib/docs/scrapers/ansible.rb +++ b/lib/docs/scrapers/ansible.rb @@ -88,10 +88,9 @@ module Docs list_of_all_modules.html) end - def get_latest_version(options, &block) - fetch_doc('https://docs.ansible.com/ansible/latest/index.html', options) do |doc| - block.call doc.at_css('.DocSiteProduct-CurrentVersion').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.ansible.com/ansible/latest/index.html', opts) + doc.at_css('.DocSiteProduct-CurrentVersion').content.strip end end end diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb index ba0fa340..1301b574 100644 --- a/lib/docs/scrapers/apache.rb +++ b/lib/docs/scrapers/apache.rb @@ -34,10 +34,9 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - fetch_doc('http://httpd.apache.org/docs/', options) do |doc| - block.call doc.at_css('#apcontents > ul a')['href'][0...-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://httpd.apache.org/docs/', opts) + doc.at_css('#apcontents > ul a')['href'][0...-1] end end end diff --git a/lib/docs/scrapers/apache_pig.rb b/lib/docs/scrapers/apache_pig.rb index 5454140b..f35085e6 100644 --- a/lib/docs/scrapers/apache_pig.rb +++ b/lib/docs/scrapers/apache_pig.rb @@ -43,11 +43,10 @@ module Docs self.base_url = "https://pig.apache.org/docs/r#{release}/" end - def get_latest_version(options, &block) - fetch_doc('https://pig.apache.org/', options) do |doc| - item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') - block.call item.content.strip.sub(/Release /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://pig.apache.org/', opts) + item = doc.at_css('div[id="menu_1.2"] > .menuitem:last-child') + item.content.strip.sub(/Release /, '') end end end diff --git a/lib/docs/scrapers/async.rb b/lib/docs/scrapers/async.rb index 18e9bbbf..67498eed 100644 --- a/lib/docs/scrapers/async.rb +++ b/lib/docs/scrapers/async.rb @@ -18,11 +18,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://caolan.github.io/async/', options) do |doc| - version = doc.at_css('#version-dropdown > a').content.strip[1..-1] - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://caolan.github.io/async/', opts) + doc.at_css('#version-dropdown > a').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb index 675f86be..c8d716f1 100644 --- a/lib/docs/scrapers/babel.rb +++ b/lib/docs/scrapers/babel.rb @@ -23,10 +23,9 @@ module Docs '
' end - def get_latest_version(options, &block) - fetch_doc('https://babeljs.io/docs/en/', options) do |doc| - block.call doc.at_css('a[href="/versions"] > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://babeljs.io/docs/en/', opts) + doc.at_css('a[href="/versions"] > h3').content end end end diff --git a/lib/docs/scrapers/backbone.rb b/lib/docs/scrapers/backbone.rb index ad6220e5..2b33505e 100644 --- a/lib/docs/scrapers/backbone.rb +++ b/lib/docs/scrapers/backbone.rb @@ -21,11 +21,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://backbonejs.org/', options) do |doc| - version = doc.at_css('.version').content - block.call version[1...-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://backbonejs.org/', opts) + doc.at_css('.version').content[1...-1] end end end diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb index 5556f5b9..92ef894a 100644 --- a/lib/docs/scrapers/bash.rb +++ b/lib/docs/scrapers/bash.rb @@ -18,11 +18,9 @@ module Docs Licensed under the GNU Free Documentation License. HTML - def get_latest_version(options, &block) - fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', options) do |body| - version = body.scan(/, Version ([0-9.]+)/)[0][0] - block.call version[0...-1] - end + def get_latest_version(opts) + body = fetch('https://www.gnu.org/software/bash/manual/html_node/index.html', opts) + body.scan(/, Version ([0-9.]+)/)[0][0][0...-1] end end end diff --git a/lib/docs/scrapers/bluebird.rb b/lib/docs/scrapers/bluebird.rb index 8a960b87..8f38120a 100644 --- a/lib/docs/scrapers/bluebird.rb +++ b/lib/docs/scrapers/bluebird.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('bluebird', options, &block) + def get_latest_version(opts) + get_npm_version('bluebird', opts) end end end diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb index aa0b4cc3..8571462e 100644 --- a/lib/docs/scrapers/bootstrap.rb +++ b/lib/docs/scrapers/bootstrap.rb @@ -35,10 +35,9 @@ module Docs options[:only] = %w(getting-started/ css/ components/ javascript/) end - def get_latest_version(options, &block) - fetch_doc('https://getbootstrap.com/', options) do |doc| - block.call doc.at_css('#bd-versions').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://getbootstrap.com/', opts) + doc.at_css('#bd-versions').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/bottle.rb b/lib/docs/scrapers/bottle.rb index 6e4a19a8..d0397ec7 100644 --- a/lib/docs/scrapers/bottle.rb +++ b/lib/docs/scrapers/bottle.rb @@ -28,11 +28,10 @@ module Docs self.base_url = "https://bottlepy.org/docs/#{self.version}/" end - def get_latest_version(options, &block) - fetch_doc('https://bottlepy.org/docs/stable/', options) do |doc| - label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') - block.call label.content.sub(/Bottle /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://bottlepy.org/docs/stable/', opts) + label = doc.at_css('.sphinxsidebarwrapper > ul > li > b') + label.content.sub(/Bottle /, '') end end end diff --git a/lib/docs/scrapers/bower.rb b/lib/docs/scrapers/bower.rb index 1102ee75..aab2a1e9 100644 --- a/lib/docs/scrapers/bower.rb +++ b/lib/docs/scrapers/bower.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('bower', options, &block) + def get_latest_version(opts) + get_npm_version('bower', opts) end end end diff --git a/lib/docs/scrapers/c.rb b/lib/docs/scrapers/c.rb index 0ab0ac39..ec99f704 100644 --- a/lib/docs/scrapers/c.rb +++ b/lib/docs/scrapers/c.rb @@ -26,12 +26,11 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| - link = doc.at_css('a[title^="File:"]') - date = link.content.scan(/(\d+)\./)[0][0] - block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i - end + def get_latest_version(opts) + doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts) + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + DateTime.strptime(date, '%Y%m%d').to_time.to_i end private diff --git a/lib/docs/scrapers/cakephp.rb b/lib/docs/scrapers/cakephp.rb index b123ab7a..6291b4ab 100644 --- a/lib/docs/scrapers/cakephp.rb +++ b/lib/docs/scrapers/cakephp.rb @@ -71,10 +71,9 @@ module Docs self.base_url = 'https://api.cakephp.org/2.7/' end - def get_latest_version(options, &block) - fetch_doc('https://api.cakephp.org/3.7/', options) do |doc| - block.call doc.at_css('.version-picker .dropdown-toggle').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://api.cakephp.org/3.7/', opts) + doc.at_css('.version-picker .dropdown-toggle').content.strip end private diff --git a/lib/docs/scrapers/chai.rb b/lib/docs/scrapers/chai.rb index 422bd5a9..759f7540 100644 --- a/lib/docs/scrapers/chai.rb +++ b/lib/docs/scrapers/chai.rb @@ -24,8 +24,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('chai', options, &block) + def get_latest_version(opts) + get_npm_version('chai', opts) end end end diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb index f9a248bd..f0b7d6b0 100644 --- a/lib/docs/scrapers/chef.rb +++ b/lib/docs/scrapers/chef.rb @@ -48,10 +48,9 @@ module Docs options[:only_patterns] = [/\A#{client_path}\//, /\A#{server_path}\//] end - def get_latest_version(options, &block) - fetch_doc('https://downloads.chef.io/chef', options) do |doc| - block.call doc.at_css('h1.product-heading > span').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://downloads.chef.io/chef', opts) + doc.at_css('h1.product-heading > span').content.strip end end end diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb index 465a4493..b5785bd3 100644 --- a/lib/docs/scrapers/clojure.rb +++ b/lib/docs/scrapers/clojure.rb @@ -28,10 +28,9 @@ module Docs self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.7.0/' end - def get_latest_version(options, &block) - fetch_doc('http://clojure.github.io/clojure/index.html', options) do |doc| - block.call doc.at_css('#header-version').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://clojure.github.io/clojure/index.html', opts) + doc.at_css('#header-version').content[1..-1] end end end diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb index dde4721c..7548a4a9 100644 --- a/lib/docs/scrapers/cmake.rb +++ b/lib/docs/scrapers/cmake.rb @@ -60,11 +60,10 @@ module Docs self.base_url = 'https://cmake.org/cmake/help/v3.5/' end - def get_latest_version(options, &block) - fetch_doc('https://cmake.org/documentation/', options) do |doc| - link = doc.at_css('.entry-content ul > li > strong > a > big') - block.call link.content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://cmake.org/documentation/', opts) + link = doc.at_css('.entry-content ul > li > strong > a > big') + link.content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb index 2e28de7f..caafc9cf 100644 --- a/lib/docs/scrapers/codeception.rb +++ b/lib/docs/scrapers/codeception.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://codeception.com/changelog', options) do |doc| - block.call doc.at_css('#page > h4').content - end + def get_latest_version(opts) + doc = fetch_doc('https://codeception.com/changelog', opts) + doc.at_css('#page > h4').content end end end diff --git a/lib/docs/scrapers/codeceptjs.rb b/lib/docs/scrapers/codeceptjs.rb index e3f4fda8..34d9b855 100644 --- a/lib/docs/scrapers/codeceptjs.rb +++ b/lib/docs/scrapers/codeceptjs.rb @@ -22,8 +22,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('codeceptjs', options, &block) + def get_latest_version(opts) + get_npm_version('codeceptjs', opts) end end end diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb index 864cf700..05258d9a 100644 --- a/lib/docs/scrapers/codeigniter.rb +++ b/lib/docs/scrapers/codeigniter.rb @@ -39,11 +39,10 @@ module Docs self.release = '3.1.8' end - def get_latest_version(options, &block) - fetch_doc('https://codeigniter.com/user_guide/changelog.html', options) do |doc| - header = doc.at_css('#change-log h2') - block.call header.content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://codeigniter.com/user_guide/changelog.html', opts) + header = doc.at_css('#change-log h2') + header.content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb index d848d208..695f3697 100644 --- a/lib/docs/scrapers/coffeescript.rb +++ b/lib/docs/scrapers/coffeescript.rb @@ -31,8 +31,8 @@ module Docs options[:container] = '.container' end - def get_latest_version(options, &block) - get_npm_version('coffeescript', options, &block) + def get_latest_version(opts) + get_npm_version('coffeescript', opts) end end end diff --git a/lib/docs/scrapers/cordova.rb b/lib/docs/scrapers/cordova.rb index efe8fb03..65cf7f60 100644 --- a/lib/docs/scrapers/cordova.rb +++ b/lib/docs/scrapers/cordova.rb @@ -43,13 +43,14 @@ module Docs self.base_url = 'https://cordova.apache.org/docs/en/6.x/' end - def get_latest_version(options, &block) - fetch_doc('https://cordova.apache.org/docs/en/latest/', options) do |doc| - label = doc.at_css('#versionDropdown').content.strip - version = label.scan(/([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://cordova.apache.org/docs/en/latest/', opts) + + label = doc.at_css('#versionDropdown').content.strip + version = label.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end end end diff --git a/lib/docs/scrapers/cpp.rb b/lib/docs/scrapers/cpp.rb index d26eae6a..f96ee8f1 100644 --- a/lib/docs/scrapers/cpp.rb +++ b/lib/docs/scrapers/cpp.rb @@ -35,12 +35,11 @@ module Docs HTML # Same as get_latest_version in lib/docs/scrapers/c.rb - def get_latest_version(options, &block) - fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', options) do |doc| - link = doc.at_css('a[title^="File:"]') - date = link.content.scan(/(\d+)\./)[0][0] - block.call DateTime.strptime(date, '%Y%m%d').to_time.to_i - end + def get_latest_version(opts) + doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts) + link = doc.at_css('a[title^="File:"]') + date = link.content.scan(/(\d+)\./)[0][0] + DateTime.strptime(date, '%Y%m%d').to_time.to_i end private diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb index e70317f2..14537c7f 100644 --- a/lib/docs/scrapers/crystal.rb +++ b/lib/docs/scrapers/crystal.rb @@ -35,10 +35,9 @@ module Docs end } - def get_latest_version(options, &block) - fetch('https://crystal-lang.org/api', options) do |body| - block.call body.scan(/Crystal Docs ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://crystal-lang.org/api', opts) + body.scan(/Crystal Docs ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/d.rb b/lib/docs/scrapers/d.rb index b0adaf31..e1475b45 100644 --- a/lib/docs/scrapers/d.rb +++ b/lib/docs/scrapers/d.rb @@ -27,10 +27,9 @@ module Docs %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html) end - def get_latest_version(options, &block) - fetch_doc('https://dlang.org/changelog/', options) do |doc| - block.call doc.at_css('#content > ul > li:nth-child(2) > a')['id'] - end + def get_latest_version(opts) + doc = fetch_doc('https://dlang.org/changelog/', opts) + doc.at_css('#content > ul > li:nth-child(2) > a')['id'] end end end diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb index cfbbafc9..e26c1f3d 100644 --- a/lib/docs/scrapers/d3.rb +++ b/lib/docs/scrapers/d3.rb @@ -59,8 +59,8 @@ module Docs options[:only_patterns] = [/\.md\z/] end - def get_latest_version(options, &block) - get_npm_version('d3', options, &block) + def get_latest_version(opts) + get_npm_version('d3', opts) end end end diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb index 42d20423..322bfe2a 100644 --- a/lib/docs/scrapers/dart.rb +++ b/lib/docs/scrapers/dart.rb @@ -32,11 +32,10 @@ module Docs self.base_url = "https://api.dartlang.org/stable/#{release}/" end - def get_latest_version(options, &block) - fetch_doc('https://api.dartlang.org/', options) do |doc| - label = doc.at_css('footer > span').content.strip - block.call label.sub(/Dart /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://api.dartlang.org/', opts) + label = doc.at_css('footer > span').content.strip + label.sub(/Dart /, '') end end end diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb index 746c0f40..6d48c6d7 100644 --- a/lib/docs/scrapers/django.rb +++ b/lib/docs/scrapers/django.rb @@ -64,10 +64,9 @@ module Docs self.base_url = 'https://docs.djangoproject.com/en/1.8/' end - def get_latest_version(options, &block) - fetch_doc('https://docs.djangoproject.com/', options) do |doc| - block.call doc.at_css('#doc-versions > li.current > span > strong').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.djangoproject.com/', opts) + doc.at_css('#doc-versions > li.current > span > strong').content end end end diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb index dd849391..3ef60aab 100644 --- a/lib/docs/scrapers/docker.rb +++ b/lib/docs/scrapers/docker.rb @@ -138,11 +138,10 @@ module Docs options[:only_patterns] << /\Aswarm\// end - def get_latest_version(options, &block) - fetch_doc('https://docs.docker.com/', options) do |doc| - label = doc.at_css('.nav-container button.dropdown-toggle').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.docker.com/', opts) + label = doc.at_css('.nav-container button.dropdown-toggle').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb index 66dccb6f..79898916 100644 --- a/lib/docs/scrapers/dojo.rb +++ b/lib/docs/scrapers/dojo.rb @@ -36,10 +36,9 @@ module Docs urls.map { |url| "#{url}" }.join end - def get_latest_version(options, &block) - fetch_doc('https://dojotoolkit.org/api/', options) do |doc| - block.call doc.at_css('#versionSelector > option[selected]').content - end + def get_latest_version(opts) + doc = fetch_doc('https://dojotoolkit.org/api/', opts) + doc.at_css('#versionSelector > option[selected]').content end private diff --git a/lib/docs/scrapers/drupal.rb b/lib/docs/scrapers/drupal.rb index 92da4193..f29b585b 100644 --- a/lib/docs/scrapers/drupal.rb +++ b/lib/docs/scrapers/drupal.rb @@ -99,13 +99,14 @@ module Docs ] end - def get_latest_version(options, &block) - fetch_doc('http://cgit.drupalcode.org/drupal', options) do |doc| - version = doc.at_css('td.form > form > select > option[selected]').content - version = version.scan(/([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('http://cgit.drupalcode.org/drupal', opts) + + version = doc.at_css('td.form > form > select > option[selected]').content + version = version.scan(/([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end end end diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb index dd3cf00a..8e635f49 100644 --- a/lib/docs/scrapers/electron.rb +++ b/lib/docs/scrapers/electron.rb @@ -23,10 +23,9 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://electronjs.org/docs', options) do |doc| - block.call doc.at_css('.docs-version').content - end + def get_latest_version(opts) + doc = fetch_doc('https://electronjs.org/docs', opts) + doc.at_css('.docs-version').content end end end diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index d5b8dbe6..a25cca41 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -98,10 +98,9 @@ module Docs ] end - def get_latest_version(options, &block) - fetch_doc('https://hexdocs.pm/elixir/api-reference.html', options) do |doc| - block.call doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts) + doc.at_css('h2.sidebar-projectVersion').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/ember.rb b/lib/docs/scrapers/ember.rb index 24a8817e..6f853bb9 100644 --- a/lib/docs/scrapers/ember.rb +++ b/lib/docs/scrapers/ember.rb @@ -57,10 +57,9 @@ module Docs ) end - def get_latest_version(options, &block) - fetch_doc('https://emberjs.com/api/ember/release', options) do |doc| - block.call doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://emberjs.com/api/ember/release', opts) + doc.at_css('.sidebar > .select-container .ember-power-select-selected-item').content.strip end end end diff --git a/lib/docs/scrapers/erlang.rb b/lib/docs/scrapers/erlang.rb index 0211da55..14a87cf5 100644 --- a/lib/docs/scrapers/erlang.rb +++ b/lib/docs/scrapers/erlang.rb @@ -56,10 +56,9 @@ module Docs self.release = '18.3' end - def get_latest_version(options, &block) - fetch_doc('https://www.erlang.org/downloads', options) do |doc| - block.call doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://www.erlang.org/downloads', opts) + doc.at_css('.col-lg-3 > ul > li').content.strip.sub(/OTP /, '') end end end diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb index dac9c283..c213eacc 100644 --- a/lib/docs/scrapers/eslint.rb +++ b/lib/docs/scrapers/eslint.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('eslint', options, &block) + def get_latest_version(opts) + get_npm_version('eslint', opts) end end end diff --git a/lib/docs/scrapers/express.rb b/lib/docs/scrapers/express.rb index 67ba07e8..990019fb 100644 --- a/lib/docs/scrapers/express.rb +++ b/lib/docs/scrapers/express.rb @@ -29,8 +29,8 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License v3.0. HTML - def get_latest_version(options, &block) - get_npm_version('express', options, &block) + def get_latest_version(opts) + get_npm_version('express', opts) end end end diff --git a/lib/docs/scrapers/falcon.rb b/lib/docs/scrapers/falcon.rb index cd5b70cd..8ba69150 100644 --- a/lib/docs/scrapers/falcon.rb +++ b/lib/docs/scrapers/falcon.rb @@ -34,10 +34,9 @@ module Docs self.base_url = "https://falcon.readthedocs.io/en/#{self.release}/" end - def get_latest_version(options, &block) - fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', options) do |doc| - block.call doc.at_css('#changelogs ul > li > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://falcon.readthedocs.io/en/stable/changes/index.html', opts) + doc.at_css('#changelogs ul > li > a').content end end end diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb index 9340961a..c9a98802 100644 --- a/lib/docs/scrapers/fish.rb +++ b/lib/docs/scrapers/fish.rb @@ -47,10 +47,9 @@ module Docs self.base_url = "https://fishshell.com/docs/#{version}/" end - def get_latest_version(options, &block) - fetch_doc('http://fishshell.com/docs/current/index.html', options) do |doc| - block.call doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://fishshell.com/docs/current/index.html', opts) + doc.at_css('#toc-index').content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb index 546473f7..b3b5a02f 100644 --- a/lib/docs/scrapers/flow.rb +++ b/lib/docs/scrapers/flow.rb @@ -19,8 +19,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('flow-bin', options, &block) + def get_latest_version(opts) + get_npm_version('flow-bin', opts) end end end diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb index f10473d0..9de5cb0d 100644 --- a/lib/docs/scrapers/git.rb +++ b/lib/docs/scrapers/git.rb @@ -20,10 +20,9 @@ module Docs Licensed under the GNU General Public License version 2. HTML - def get_latest_version(options, &block) - fetch_doc('https://git-scm.com/', options) do |doc| - block.call doc.at_css('.version').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://git-scm.com/', opts) + doc.at_css('.version').content.strip end end end diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb index 3252dd6d..565706d9 100644 --- a/lib/docs/scrapers/gnu/gcc.rb +++ b/lib/docs/scrapers/gnu/gcc.rb @@ -100,11 +100,10 @@ module Docs options[:replace_paths] = CPP_PATHS end - def get_latest_version(options, &block) - fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| - label = doc.at_css('ul > li > ul > li > a').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts) + label = doc.at_css('ul > li > ul > li > a').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb index f72f7d65..dd18827c 100644 --- a/lib/docs/scrapers/gnu/gnu_fortran.rb +++ b/lib/docs/scrapers/gnu/gnu_fortran.rb @@ -26,11 +26,10 @@ module Docs self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/" end - def get_latest_version(options, &block) - fetch_doc('https://gcc.gnu.org/onlinedocs/', options) do |doc| - label = doc.at_css('ul > li > ul > li > a').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://gcc.gnu.org/onlinedocs/', opts) + label = doc.at_css('ul > li > ul > li > a').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb index 6f8f7a4a..8997b7b1 100644 --- a/lib/docs/scrapers/go.rb +++ b/lib/docs/scrapers/go.rb @@ -24,13 +24,14 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://golang.org/pkg/', options) do |doc| - footer = doc.at_css('#footer').content - version = footer.scan(/go([0-9.]+)/)[0][0] - version = version[0...-1] if version.end_with?('.') - block.call version - end + def get_latest_version(opts) + doc = fetch_doc('https://golang.org/pkg/', opts) + + footer = doc.at_css('#footer').content + version = footer.scan(/go([0-9.]+)/)[0][0] + version = version[0...-1] if version.end_with?('.') + + version end private diff --git a/lib/docs/scrapers/godot.rb b/lib/docs/scrapers/godot.rb index d43782c2..06c330b2 100644 --- a/lib/docs/scrapers/godot.rb +++ b/lib/docs/scrapers/godot.rb @@ -38,10 +38,9 @@ module Docs self.base_url = "http://docs.godotengine.org/en/#{self.version}/" end - def get_latest_version(options, &block) - fetch_doc('https://docs.godotengine.org/', options) do |doc| - block.call doc.at_css('.version').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.godotengine.org/', opts) + doc.at_css('.version').content.strip end end end diff --git a/lib/docs/scrapers/graphite.rb b/lib/docs/scrapers/graphite.rb index d1d8b9d1..83e9314a 100644 --- a/lib/docs/scrapers/graphite.rb +++ b/lib/docs/scrapers/graphite.rb @@ -18,10 +18,9 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', options) do |doc| - block.call doc.at_css('#release-notes li > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://graphite.readthedocs.io/en/latest/releases.html', opts) + doc.at_css('#release-notes li > a').content end end end diff --git a/lib/docs/scrapers/grunt.rb b/lib/docs/scrapers/grunt.rb index 1e8af9fb..469d10a0 100644 --- a/lib/docs/scrapers/grunt.rb +++ b/lib/docs/scrapers/grunt.rb @@ -27,8 +27,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('grunt-cli', options, &block) + def get_latest_version(opts) + get_npm_version('grunt-cli', opts) end end end diff --git a/lib/docs/scrapers/handlebars.rb b/lib/docs/scrapers/handlebars.rb index 7df63102..046cdf0f 100644 --- a/lib/docs/scrapers/handlebars.rb +++ b/lib/docs/scrapers/handlebars.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('handlebars', options, &block) + def get_latest_version(opts) + get_npm_version('handlebars', opts) end end end diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb index fc848a7a..fb118851 100755 --- a/lib/docs/scrapers/haskell.rb +++ b/lib/docs/scrapers/haskell.rb @@ -69,12 +69,11 @@ module Docs options[:only_patterns] = [/\Alibraries\//] end - def get_latest_version(options, &block) - fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', options) do |doc| - links = doc.css('a').to_a - versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)} - block.call versions.find {|version| !version.empty?}[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://downloads.haskell.org/~ghc/latest/docs/html/', opts) + links = doc.css('a').to_a + versions = links.map {|link| link['href'].scan(/ghc-([0-9.]+)/)} + versions.find {|version| !version.empty?}[0][0] end end end diff --git a/lib/docs/scrapers/haxe.rb b/lib/docs/scrapers/haxe.rb index 5a685efc..2dbab01a 100644 --- a/lib/docs/scrapers/haxe.rb +++ b/lib/docs/scrapers/haxe.rb @@ -67,11 +67,10 @@ module Docs self.base_url = 'https://api.haxe.org/python/' end - def get_latest_version(options, &block) - fetch_doc('https://api.haxe.org/', options) do |doc| - label = doc.at_css('.container.main-content h1 > small').content - block.call label.sub(/version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://api.haxe.org/', opts) + label = doc.at_css('.container.main-content h1 > small').content + label.sub(/version /, '') end end end diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb index fef1ed05..c5647709 100644 --- a/lib/docs/scrapers/homebrew.rb +++ b/lib/docs/scrapers/homebrew.rb @@ -20,10 +20,8 @@ module Docs Licensed under the BSD 2-Clause License. HTML - def get_latest_version(options, &block) - get_latest_github_release('Homebrew', 'brew', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('Homebrew', 'brew', opts)['name'] end end end diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb index 342ce107..8b1b47a2 100644 --- a/lib/docs/scrapers/immutable.rb +++ b/lib/docs/scrapers/immutable.rb @@ -55,8 +55,8 @@ module Docs capybara.html end - def get_latest_version(options, &block) - get_npm_version('immutable', options, &block) + def get_latest_version(opts) + get_npm_version('immutable', opts) end end end diff --git a/lib/docs/scrapers/influxdata.rb b/lib/docs/scrapers/influxdata.rb index 4fc98c16..db160f9c 100644 --- a/lib/docs/scrapers/influxdata.rb +++ b/lib/docs/scrapers/influxdata.rb @@ -47,11 +47,10 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://docs.influxdata.com/influxdb/', options) do |doc| - label = doc.at_css('.navbar--current-product').content.strip - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.influxdata.com/influxdb/', opts) + label = doc.at_css('.navbar--current-product').content.strip + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb index 5f38e3d5..14c51869 100644 --- a/lib/docs/scrapers/jasmine.rb +++ b/lib/docs/scrapers/jasmine.rb @@ -18,10 +18,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('jasmine', 'jasmine', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('jasmine', 'jasmine', opts)['name'] end end end diff --git a/lib/docs/scrapers/jekyll.rb b/lib/docs/scrapers/jekyll.rb index a6af352f..500eee10 100644 --- a/lib/docs/scrapers/jekyll.rb +++ b/lib/docs/scrapers/jekyll.rb @@ -29,10 +29,9 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - fetch_doc('https://jekyllrb.com/docs/', options) do |doc| - block.call doc.at_css('.meta a').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://jekyllrb.com/docs/', opts) + doc.at_css('.meta a').content[1..-1] end end end diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb index 71efcf54..a495d939 100644 --- a/lib/docs/scrapers/jest.rb +++ b/lib/docs/scrapers/jest.rb @@ -18,10 +18,9 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://jestjs.io/docs/en/getting-started', options) do |doc| - block.call doc.at_css('header > a > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://jestjs.io/docs/en/getting-started', opts) + doc.at_css('header > a > h3').content end end end diff --git a/lib/docs/scrapers/jquery/jquery_core.rb b/lib/docs/scrapers/jquery/jquery_core.rb index dad609e7..a0c8b97a 100644 --- a/lib/docs/scrapers/jquery/jquery_core.rb +++ b/lib/docs/scrapers/jquery/jquery_core.rb @@ -23,8 +23,8 @@ module Docs /index/i ] - def get_latest_version(options, &block) - get_npm_version('jquery', options, &block) + def get_latest_version(opts) + get_npm_version('jquery', opts) end end end diff --git a/lib/docs/scrapers/jquery/jquery_mobile.rb b/lib/docs/scrapers/jquery/jquery_mobile.rb index 53b2c624..5b856a95 100644 --- a/lib/docs/scrapers/jquery/jquery_mobile.rb +++ b/lib/docs/scrapers/jquery/jquery_mobile.rb @@ -17,11 +17,9 @@ module Docs url.sub! 'http://api.jquerymobile.com/', 'https://api.jquerymobile.com/' end - def get_latest_version(options, &block) - fetch_doc('https://jquerymobile.com/', options) do |doc| - label = doc.at_css('.download-box > .download-option:last-child > span').content - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://jquerymobile.com/', opts) + doc.at_css('.download-box > .download-option:last-child > span').content.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb index 05c276e1..021d1d22 100644 --- a/lib/docs/scrapers/jquery/jquery_ui.rb +++ b/lib/docs/scrapers/jquery/jquery_ui.rb @@ -16,8 +16,8 @@ module Docs url.sub! 'http://api.jqueryui.com/', 'https://api.jqueryui.com/' end - def get_latest_version(options, &block) - get_npm_version('jquery-ui', options, &block) + def get_latest_version(opts) + get_npm_version('jquery-ui', opts) end end end diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb index 39feca71..d88d46b6 100644 --- a/lib/docs/scrapers/jsdoc.rb +++ b/lib/docs/scrapers/jsdoc.rb @@ -22,10 +22,8 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('jsdoc3', 'jsdoc', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('jsdoc3', 'jsdoc', opts)['tag_name'] end end end diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb index 0875835a..d152f05d 100644 --- a/lib/docs/scrapers/julia.rb +++ b/lib/docs/scrapers/julia.rb @@ -50,10 +50,8 @@ module Docs html_filters.push 'julia/entries_sphinx', 'julia/clean_html_sphinx', 'sphinx/clean_html' end - def get_latest_version(options, &block) - get_latest_github_release('JuliaLang', 'julia', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('JuliaLang', 'julia', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/knockout.rb b/lib/docs/scrapers/knockout.rb index 60af1540..6556bca8 100644 --- a/lib/docs/scrapers/knockout.rb +++ b/lib/docs/scrapers/knockout.rb @@ -34,10 +34,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('knockout', 'knockout', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('knockout', 'knockout', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/koa.rb b/lib/docs/scrapers/koa.rb index 4d90e30f..cac14920 100644 --- a/lib/docs/scrapers/koa.rb +++ b/lib/docs/scrapers/koa.rb @@ -35,8 +35,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('koa', options, &block) + def get_latest_version(opts) + get_npm_version('koa', opts) end end end diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb index 7539212d..5f508ae7 100644 --- a/lib/docs/scrapers/kotlin.rb +++ b/lib/docs/scrapers/kotlin.rb @@ -29,10 +29,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('JetBrains', 'kotlin', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('JetBrains', 'kotlin', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index cdf32732..4fc17368 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -134,10 +134,8 @@ module Docs end end - def get_latest_version(options, &block) - get_latest_github_release('laravel', 'laravel', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('laravel', 'laravel', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb index 24bc6142..38e497e7 100644 --- a/lib/docs/scrapers/leaflet.rb +++ b/lib/docs/scrapers/leaflet.rb @@ -39,11 +39,10 @@ module Docs self.base_url = "https://leafletjs.com/reference-#{release}.html" end - def get_latest_version(options, &block) - fetch_doc('https://leafletjs.com/index.html', options) do |doc| - link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first - block.call link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://leafletjs.com/index.html', opts) + link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first + link['href'].scan(/reference-([0-9.]+)\.html/)[0][0] end end end diff --git a/lib/docs/scrapers/less.rb b/lib/docs/scrapers/less.rb index 00c884eb..b19bbe17 100644 --- a/lib/docs/scrapers/less.rb +++ b/lib/docs/scrapers/less.rb @@ -22,11 +22,10 @@ module Docs Licensed under the Creative Commons Attribution License 3.0. HTML - def get_latest_version(options, &block) - fetch_doc('http://lesscss.org/features/', options) do |doc| - label = doc.at_css('.footer-links > li').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://lesscss.org/features/', opts) + label = doc.at_css('.footer-links > li').content + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/liquid.rb b/lib/docs/scrapers/liquid.rb index 4630b2d1..b8e40d59 100644 --- a/lib/docs/scrapers/liquid.rb +++ b/lib/docs/scrapers/liquid.rb @@ -20,10 +20,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_tags('Shopify', 'liquid', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('Shopify', 'liquid', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/lodash.rb b/lib/docs/scrapers/lodash.rb index 5488b9ab..bce625e6 100644 --- a/lib/docs/scrapers/lodash.rb +++ b/lib/docs/scrapers/lodash.rb @@ -33,10 +33,9 @@ module Docs self.base_url = "https://lodash.com/docs/#{release}" end - def get_latest_version(options, &block) - fetch_doc('https://lodash.com/docs/', options) do |doc| - block.call doc.at_css('#version > option[selected]').content - end + def get_latest_version(opts) + doc = fetch_doc('https://lodash.com/docs/', opts) + doc.at_css('#version > option[selected]').content end end end diff --git a/lib/docs/scrapers/love.rb b/lib/docs/scrapers/love.rb index 019edbab..887b796f 100644 --- a/lib/docs/scrapers/love.rb +++ b/lib/docs/scrapers/love.rb @@ -40,10 +40,9 @@ module Docs Licensed under the GNU Free Documentation License, Version 1.3. HTML - def get_latest_version(options, &block) - fetch_doc('https://love2d.org/wiki/Version_History', options) do |doc| - block.call doc.at_css('#mw-content-text table a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://love2d.org/wiki/Version_History', opts) + doc.at_css('#mw-content-text table a').content end end end diff --git a/lib/docs/scrapers/lua.rb b/lib/docs/scrapers/lua.rb index 30af5523..e3608918 100644 --- a/lib/docs/scrapers/lua.rb +++ b/lib/docs/scrapers/lua.rb @@ -27,10 +27,9 @@ module Docs self.base_url = 'https://www.lua.org/manual/5.1/' end - def get_latest_version(options, &block) - fetch_doc('https://www.lua.org/manual/', options) do |doc| - block.call doc.at_css('p.menubar > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://www.lua.org/manual/', opts) + doc.at_css('p.menubar > a').content end end end diff --git a/lib/docs/scrapers/marionette.rb b/lib/docs/scrapers/marionette.rb index 12de6d0c..fd1eab8e 100644 --- a/lib/docs/scrapers/marionette.rb +++ b/lib/docs/scrapers/marionette.rb @@ -39,8 +39,8 @@ module Docs html_filters.push 'marionette/entries_v2' end - def get_latest_version(options, &block) - get_npm_version('backbone.marionette', options, &block) + def get_latest_version(opts) + get_npm_version('backbone.marionette', opts) end end end diff --git a/lib/docs/scrapers/markdown.rb b/lib/docs/scrapers/markdown.rb index 3400e270..b837c692 100644 --- a/lib/docs/scrapers/markdown.rb +++ b/lib/docs/scrapers/markdown.rb @@ -14,8 +14,8 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - block.call '1.0.0' + def get_latest_version(opts) + '1.0.0' end end end diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb index 948955a6..4a882270 100644 --- a/lib/docs/scrapers/matplotlib.rb +++ b/lib/docs/scrapers/matplotlib.rb @@ -65,10 +65,8 @@ module Docs ] end - def get_latest_version(options, &block) - get_latest_github_release('matplotlib', 'matplotlib', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('matplotlib', 'matplotlib', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb index ccb27af9..defb4533 100644 --- a/lib/docs/scrapers/mdn/mdn.rb +++ b/lib/docs/scrapers/mdn/mdn.rb @@ -21,10 +21,9 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later. HTML - def get_latest_version(opts, &block) - fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts) do |json| - block.call DateTime.parse(json[0]['pubdate']).to_time.to_i - end + def get_latest_version(opts) + json = fetch_json("https://developer.mozilla.org/en-US/docs/feeds/json/tag/#{options[:mdn_tag]}", opts) + DateTime.parse(json[0]['pubdate']).to_time.to_i end private diff --git a/lib/docs/scrapers/meteor.rb b/lib/docs/scrapers/meteor.rb index 02b81bc3..a758d154 100644 --- a/lib/docs/scrapers/meteor.rb +++ b/lib/docs/scrapers/meteor.rb @@ -46,10 +46,9 @@ module Docs options[:fix_urls] = nil end - def get_latest_version(options, &block) - fetch_doc('https://docs.meteor.com/#/full/', options) do |doc| - block.call doc.at_css('select.version-select > option').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.meteor.com/#/full/', opts) + doc.at_css('select.version-select > option').content end end end diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb index 6654d754..04945425 100644 --- a/lib/docs/scrapers/mocha.rb +++ b/lib/docs/scrapers/mocha.rb @@ -19,8 +19,8 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International License. HTML - def get_latest_version(options, &block) - get_npm_version('mocha', options, &block) + def get_latest_version(opts) + get_npm_version('mocha', opts) end end end diff --git a/lib/docs/scrapers/modernizr.rb b/lib/docs/scrapers/modernizr.rb index 93a738bb..01ad49a7 100644 --- a/lib/docs/scrapers/modernizr.rb +++ b/lib/docs/scrapers/modernizr.rb @@ -16,8 +16,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('modernizr', options, &block) + def get_latest_version(opts) + get_npm_version('modernizr', opts) end end end diff --git a/lib/docs/scrapers/moment.rb b/lib/docs/scrapers/moment.rb index 9dd27107..5b7491ea 100644 --- a/lib/docs/scrapers/moment.rb +++ b/lib/docs/scrapers/moment.rb @@ -23,10 +23,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('http://momentjs.com/', options) do |doc| - block.call doc.at_css('.hero-title > h1 > span').content - end + def get_latest_version(opts) + doc = fetch_doc('http://momentjs.com/', opts) + doc.at_css('.hero-title > h1 > span').content end end end diff --git a/lib/docs/scrapers/mongoose.rb b/lib/docs/scrapers/mongoose.rb index 2d221990..fbd4ca92 100644 --- a/lib/docs/scrapers/mongoose.rb +++ b/lib/docs/scrapers/mongoose.rb @@ -27,11 +27,10 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://mongoosejs.com/docs/', options) do |doc| - label = doc.at_css('.pure-menu-link').content.strip - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://mongoosejs.com/docs/', opts) + label = doc.at_css('.pure-menu-link').content.strip + label.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb index 07a4565f..5354b8bd 100644 --- a/lib/docs/scrapers/nginx.rb +++ b/lib/docs/scrapers/nginx.rb @@ -26,11 +26,10 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://nginx.org/en/download.html', options) do |doc| - table = doc.at_css('#content > table').inner_html - block.call table.scan(/nginx-([0-9.]+) table').inner_html + table.scan(/nginx-([0-9.]+) .docinfo > tbody > tr:last-child > td').content.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://nim-lang.org/docs/overview.html', opts) + doc.at_css('.container > .docinfo > tbody > tr:last-child > td').content.strip end end end diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb index 36c14e9e..0e5ee8fa 100644 --- a/lib/docs/scrapers/node.rb +++ b/lib/docs/scrapers/node.rb @@ -47,10 +47,9 @@ module Docs self.base_url = 'https://nodejs.org/dist/latest-v4.x/docs/api/' end - def get_latest_version(options, &block) - fetch_doc('https://nodejs.org/en/', options) do |doc| - block.call doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://nodejs.org/en/', opts) + doc.at_css('#home-intro > .home-downloadblock:last-of-type > a')['data-version'][1..-1] end end end diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb index aca5d10c..9da5daf4 100644 --- a/lib/docs/scrapers/nokogiri2.rb +++ b/lib/docs/scrapers/nokogiri2.rb @@ -20,10 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('sparklemotion', 'nokogiri', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('sparklemotion', 'nokogiri', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb index 7ef727c0..e18531ab 100644 --- a/lib/docs/scrapers/npm.rb +++ b/lib/docs/scrapers/npm.rb @@ -30,10 +30,8 @@ module Docs npm is a trademark of npm, Inc. HTML - def get_latest_version(options, &block) - get_latest_github_release('npm', 'cli', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('npm', 'cli', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb index 1fcc05d6..636fcf4a 100644 --- a/lib/docs/scrapers/numpy.rb +++ b/lib/docs/scrapers/numpy.rb @@ -50,10 +50,8 @@ module Docs self.base_url = "https://docs.scipy.org/doc/numpy-#{self.release}/reference/" end - def get_latest_version(options, &block) - get_latest_github_release('numpy', 'numpy', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('numpy', 'numpy', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb index a56a2928..c26bce4c 100644 --- a/lib/docs/scrapers/openjdk.rb +++ b/lib/docs/scrapers/openjdk.rb @@ -87,7 +87,7 @@ module Docs File.read(path).force_encoding('iso-8859-1').encode('utf-8') rescue nil end - def get_latest_version(options, &block) + def get_latest_version(opts) latest_version = 8 current_attempt = latest_version attempts = 0 @@ -95,17 +95,16 @@ module Docs while attempts < 3 current_attempt += 1 - fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", options) do |doc| - if doc.at_css('.perror').nil? - latest_version = current_attempt - attempts = 0 - else - attempts += 1 - end + doc = fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", opts) + if doc.at_css('.perror').nil? + latest_version = current_attempt + attempts = 0 + else + attempts += 1 end end - block.call latest_version + latest_version end end end diff --git a/lib/docs/scrapers/opentsdb.rb b/lib/docs/scrapers/opentsdb.rb index 54a77017..1de40478 100644 --- a/lib/docs/scrapers/opentsdb.rb +++ b/lib/docs/scrapers/opentsdb.rb @@ -19,10 +19,8 @@ module Docs Licensed under the GNU LGPLv2.1+ and GPLv3+ licenses. HTML - def get_latest_version(options, &block) - get_latest_github_release('OpenTSDB', 'opentsdb', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('OpenTSDB', 'opentsdb', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/padrino.rb b/lib/docs/scrapers/padrino.rb index 78588ef4..d34b7db5 100644 --- a/lib/docs/scrapers/padrino.rb +++ b/lib/docs/scrapers/padrino.rb @@ -24,10 +24,8 @@ module Docs request_one(url_for('index')).body end - def get_latest_version(options, &block) - get_github_tags('padrino', 'padrino-framework', options) do |tags| - block.call tags[0]['name'] - end + def get_latest_version(opts) + get_github_tags('padrino', 'padrino-framework', opts)[0]['name'] end end end diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb index f50c4427..a8c2ea51 100644 --- a/lib/docs/scrapers/pandas.rb +++ b/lib/docs/scrapers/pandas.rb @@ -50,11 +50,10 @@ module Docs self.base_url = "http://pandas.pydata.org/pandas-docs/version/#{self.release}/" end - def get_latest_version(options, &block) - fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', options) do |doc| - label = doc.at_css('.body > .section > p').content - block.call label.scan(/Version: ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('http://pandas.pydata.org/pandas-docs/stable/', opts) + label = doc.at_css('.body > .section > p').content + label.scan(/Version: ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb index 136a6d75..7e5ed8f8 100644 --- a/lib/docs/scrapers/perl.rb +++ b/lib/docs/scrapers/perl.rb @@ -44,10 +44,9 @@ module Docs self.base_url = "https://perldoc.perl.org/#{self.release}/" end - def get_latest_version(options, &block) - fetch('https://perldoc.perl.org/static/perlversion.js', options) do |body| - block.call body.scan(/>Perl ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://perldoc.perl.org/static/perlversion.js', opts) + body.scan(/>Perl ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/phalcon.rb b/lib/docs/scrapers/phalcon.rb index 82a9cf85..dd476329 100644 --- a/lib/docs/scrapers/phalcon.rb +++ b/lib/docs/scrapers/phalcon.rb @@ -30,10 +30,9 @@ module Docs self.base_url = 'https://docs.phalconphp.com/en/2.0.0/' end - def get_latest_version(options, &block) - fetch_doc('https://docs.phalconphp.com/', options) do |doc| - block.call doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.phalconphp.com/', opts) + doc.at_css('.custom-select__version').content.strip.sub(/Version /, '') end end end diff --git a/lib/docs/scrapers/phaser.rb b/lib/docs/scrapers/phaser.rb index ef265e2e..1939b1d0 100644 --- a/lib/docs/scrapers/phaser.rb +++ b/lib/docs/scrapers/phaser.rb @@ -26,10 +26,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_latest_github_release('photonstorm', 'phaser', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('photonstorm', 'phaser', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/phoenix.rb b/lib/docs/scrapers/phoenix.rb index 08948b3c..2ad053e2 100644 --- a/lib/docs/scrapers/phoenix.rb +++ b/lib/docs/scrapers/phoenix.rb @@ -47,10 +47,9 @@ module Docs end } - def get_latest_version(options, &block) - fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', options) do |doc| - block.call doc.at_css('.sidebar-projectVersion').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://hexdocs.pm/phoenix/Phoenix.html', opts) + doc.at_css('.sidebar-projectVersion').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb index b6aac039..181d8b67 100644 --- a/lib/docs/scrapers/php.rb +++ b/lib/docs/scrapers/php.rb @@ -67,11 +67,10 @@ module Docs Licensed under the Creative Commons Attribution License v3.0 or later. HTML - def get_latest_version(options, &block) - fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', options) do |doc| - label = doc.at_css('tbody.gen-changelog > tr > td').content - block.call label.split(',').last.strip - end + def get_latest_version(opts) + doc = fetch_doc('https://secure.php.net/manual/en/doc.changelog.php', opts) + label = doc.at_css('tbody.gen-changelog > tr > td').content + label.split(',').last.strip end end end diff --git a/lib/docs/scrapers/phpunit.rb b/lib/docs/scrapers/phpunit.rb index c5e8c396..5f2cbfea 100644 --- a/lib/docs/scrapers/phpunit.rb +++ b/lib/docs/scrapers/phpunit.rb @@ -38,11 +38,10 @@ module Docs self.base_url = "https://phpunit.de/manual/#{release}/en/" end - def get_latest_version(options, &block) - fetch_doc('https://phpunit.readthedocs.io/', options) do |doc| - label = doc.at_css('.rst-current-version').content.strip - block.call label.scan(/v: ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://phpunit.readthedocs.io/', opts) + label = doc.at_css('.rst-current-version').content.strip + label.scan(/v: ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb index 4946066b..cc7a85c8 100644 --- a/lib/docs/scrapers/postgresql.rb +++ b/lib/docs/scrapers/postgresql.rb @@ -81,11 +81,10 @@ module Docs html_filters.insert_before 'postgresql/extract_metadata', 'postgresql/normalize_class_names' end - def get_latest_version(options, &block) - fetch_doc('https://www.postgresql.org/docs/current/index.html', options) do |doc| - label = doc.at_css('#pgContentWrap h1.title').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://www.postgresql.org/docs/current/index.html', opts) + label = doc.at_css('#pgContentWrap h1.title').content + label.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/pug.rb b/lib/docs/scrapers/pug.rb index 9084feb1..79a1b0a8 100644 --- a/lib/docs/scrapers/pug.rb +++ b/lib/docs/scrapers/pug.rb @@ -18,8 +18,8 @@ module Docs Licensed under the MIT license. HTML - def get_latest_version(options, &block) - get_npm_version('pug', options, &block) + def get_latest_version(opts) + get_npm_version('pug', opts) end private diff --git a/lib/docs/scrapers/puppeteer.rb b/lib/docs/scrapers/puppeteer.rb index a1506c7a..043380d5 100644 --- a/lib/docs/scrapers/puppeteer.rb +++ b/lib/docs/scrapers/puppeteer.rb @@ -15,10 +15,9 @@ module Docs Licensed under the Apache License 2.0. HTML - def get_latest_version(options, &block) - get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', options) do |contents| - block.call contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('GoogleChrome', 'puppeteer', 'README.md', opts) + contents.scan(/\/v([0-9.]+)\/docs\/api\.md/)[0][0] end end end diff --git a/lib/docs/scrapers/pygame.rb b/lib/docs/scrapers/pygame.rb index 538c0722..d5a5581d 100644 --- a/lib/docs/scrapers/pygame.rb +++ b/lib/docs/scrapers/pygame.rb @@ -18,10 +18,8 @@ module Docs Licensed under the GNU LGPL License version 2.1. HTML - def get_latest_version(options, &block) - get_latest_github_release('pygame', 'pygame', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('pygame', 'pygame', opts)['tag_name'] end end end diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb index 16ecc366..c7905591 100644 --- a/lib/docs/scrapers/python.rb +++ b/lib/docs/scrapers/python.rb @@ -51,10 +51,9 @@ module Docs html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html' end - def get_latest_version(options, &block) - fetch_doc('https://docs.python.org/', options) do |doc| - block.call doc.at_css('.version_switcher_placeholder').content - end + def get_latest_version(opts) + doc = fetch_doc('https://docs.python.org/', opts) + doc.at_css('.version_switcher_placeholder').content end end end diff --git a/lib/docs/scrapers/q.rb b/lib/docs/scrapers/q.rb index 0a160488..a4c449c0 100644 --- a/lib/docs/scrapers/q.rb +++ b/lib/docs/scrapers/q.rb @@ -20,8 +20,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('q', options, &block) + def get_latest_version(opts) + get_npm_version('q', opts) end end end diff --git a/lib/docs/scrapers/qt.rb b/lib/docs/scrapers/qt.rb index 41c835e9..a1098b20 100644 --- a/lib/docs/scrapers/qt.rb +++ b/lib/docs/scrapers/qt.rb @@ -118,11 +118,9 @@ module Docs self.base_url = 'https://doc.qt.io/qt-5.6/' end - def get_latest_version(options, &block) - fetch_doc('https://doc.qt.io/qt-5/index.html', options) do |doc| - label = doc.at_css('.mainContent h1.title').content - block.call label.sub(/Qt /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://doc.qt.io/qt-5/index.html', opts) + doc.at_css('.mainContent h1.title').content.sub(/Qt /, '') end end end diff --git a/lib/docs/scrapers/ramda.rb b/lib/docs/scrapers/ramda.rb index d3751e38..0b86e365 100644 --- a/lib/docs/scrapers/ramda.rb +++ b/lib/docs/scrapers/ramda.rb @@ -16,10 +16,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://ramdajs.com/docs/', options) do |doc| - block.call doc.at_css('.navbar-brand > .version').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://ramdajs.com/docs/', opts) + doc.at_css('.navbar-brand > .version').content[1..-1] end end end diff --git a/lib/docs/scrapers/rdoc/minitest.rb b/lib/docs/scrapers/rdoc/minitest.rb index f676010d..2a4249fc 100644 --- a/lib/docs/scrapers/rdoc/minitest.rb +++ b/lib/docs/scrapers/rdoc/minitest.rb @@ -22,10 +22,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', options) do |contents| - block.call contents.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('seattlerb', 'minitest', 'History.rdoc', opts) + contents.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb index 9cb2ab9b..771c2f64 100644 --- a/lib/docs/scrapers/rdoc/rails.rb +++ b/lib/docs/scrapers/rdoc/rails.rb @@ -94,10 +94,8 @@ module Docs self.release = '4.1.16' end - def get_latest_version(options, &block) - get_latest_github_release('rails', 'rails', options) do |release| - block.call release['name'] - end + def get_latest_version(opts) + get_latest_github_release('rails', 'rails', opts)['name'] end end end diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb index 292540db..bc064660 100644 --- a/lib/docs/scrapers/rdoc/ruby.rb +++ b/lib/docs/scrapers/rdoc/ruby.rb @@ -85,14 +85,13 @@ module Docs self.release = '2.2.10' end - def get_latest_version(options, &block) - get_github_tags('ruby', 'ruby', options) do |tags| - tags.each do |tag| - version = tag['name'].gsub(/_/, '.')[1..-1] - if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 - block.call version - break - end + def get_latest_version(opts) + tags = get_github_tags('ruby', 'ruby', opts) + tags.each do |tag| + version = tag['name'].gsub(/_/, '.')[1..-1] + + if !/^([0-9.]+)$/.match(version).nil? && version.count('.') == 2 + return version end end end diff --git a/lib/docs/scrapers/react.rb b/lib/docs/scrapers/react.rb index 54038244..3c41ea5a 100644 --- a/lib/docs/scrapers/react.rb +++ b/lib/docs/scrapers/react.rb @@ -31,10 +31,9 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International Public License. HTML - def get_latest_version(options, &block) - fetch_doc('https://reactjs.org/docs/getting-started.html', options) do |doc| - block.call doc.at_css('a[href="/versions"]').content.strip[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://reactjs.org/docs/getting-started.html', opts) + doc.at_css('a[href="/versions"]').content.strip[1..-1] end end end diff --git a/lib/docs/scrapers/react_native.rb b/lib/docs/scrapers/react_native.rb index ac5bd250..fe87e492 100644 --- a/lib/docs/scrapers/react_native.rb +++ b/lib/docs/scrapers/react_native.rb @@ -31,10 +31,9 @@ module Docs Licensed under the Creative Commons Attribution 4.0 International Public License. HTML - def get_latest_version(options, &block) - fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', options) do |doc| - block.call doc.at_css('header > a > h3').content - end + def get_latest_version(opts) + doc = fetch_doc('https://facebook.github.io/react-native/docs/getting-started.html', opts) + doc.at_css('header > a > h3').content end end end diff --git a/lib/docs/scrapers/redis.rb b/lib/docs/scrapers/redis.rb index 89a67598..81440a82 100644 --- a/lib/docs/scrapers/redis.rb +++ b/lib/docs/scrapers/redis.rb @@ -20,11 +20,10 @@ module Docs Licensed under the Creative Commons Attribution-ShareAlike License 4.0. HTML - def get_latest_version(options, &block) - fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', options) do |body| - body = body.lines[1..-1].join - block.call body.scan(/Redis ([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('http://download.redis.io/redis-stable/00-RELEASENOTES', opts) + body = body.lines[1..-1].join + body.scan(/Redis ([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/redux.rb b/lib/docs/scrapers/redux.rb index 09738bb3..14f8e8b4 100644 --- a/lib/docs/scrapers/redux.rb +++ b/lib/docs/scrapers/redux.rb @@ -21,8 +21,8 @@ module Docs request_one('http://redux.js.org/index.html').body end - def get_latest_version(options, &block) - get_npm_version('redux', options, &block) + def get_latest_version(opts) + get_npm_version('redux', opts) end end end diff --git a/lib/docs/scrapers/relay.rb b/lib/docs/scrapers/relay.rb index 90423ac6..807d6e1d 100644 --- a/lib/docs/scrapers/relay.rb +++ b/lib/docs/scrapers/relay.rb @@ -19,10 +19,9 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('http://facebook.github.io/relay/en/', options) do |doc| - block.call doc.at_css('header > a > h3').content[1..-1] - end + def get_latest_version(opts) + doc = fetch_doc('http://facebook.github.io/relay/en/', opts) + doc.at_css('header > a > h3').content[1..-1] end end end diff --git a/lib/docs/scrapers/requirejs.rb b/lib/docs/scrapers/requirejs.rb index cf417a3e..80ecc722 100644 --- a/lib/docs/scrapers/requirejs.rb +++ b/lib/docs/scrapers/requirejs.rb @@ -31,8 +31,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('requirejs', options, &block) + def get_latest_version(opts) + get_npm_version('requirejs', opts) end end end diff --git a/lib/docs/scrapers/rethinkdb.rb b/lib/docs/scrapers/rethinkdb.rb index 27d913e0..3a6b87cf 100644 --- a/lib/docs/scrapers/rethinkdb.rb +++ b/lib/docs/scrapers/rethinkdb.rb @@ -58,10 +58,8 @@ module Docs CODE end - def get_latest_version(options, &block) - get_latest_github_release('rethinkdb', 'rethinkdb', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('rethinkdb', 'rethinkdb', opts)['tag_name'][1..-1] end private diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb index 7c8d1519..635afc1c 100644 --- a/lib/docs/scrapers/rust.rb +++ b/lib/docs/scrapers/rust.rb @@ -39,11 +39,10 @@ module Docs Licensed under the Apache License, Version 2.0 or the MIT license, at your option. HTML - def get_latest_version(options, &block) - fetch_doc('https://www.rust-lang.org/', options) do |doc| - label = doc.at_css('.button-download + p > a').content - block.call label.sub(/Version /, '') - end + def get_latest_version(opts) + doc = fetch_doc('https://www.rust-lang.org/', opts) + label = doc.at_css('.button-download + p > a').content + label.sub(/Version /, '') end private diff --git a/lib/docs/scrapers/sass.rb b/lib/docs/scrapers/sass.rb index f493289b..228a5337 100644 --- a/lib/docs/scrapers/sass.rb +++ b/lib/docs/scrapers/sass.rb @@ -24,10 +24,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_github_file_contents('sass', 'sass', 'VERSION', options) do |contents| - block.call contents.strip - end + def get_latest_version(opts) + get_github_file_contents('sass', 'sass', 'VERSION', opts).strip end end end diff --git a/lib/docs/scrapers/scikit_image.rb b/lib/docs/scrapers/scikit_image.rb index 3c454bcb..a5959581 100644 --- a/lib/docs/scrapers/scikit_image.rb +++ b/lib/docs/scrapers/scikit_image.rb @@ -21,10 +21,9 @@ module Docs Licensed under the BSD 3-clause License. HTML - def get_latest_version(options, &block) - get_github_tags('scikit-image', 'scikit-image', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('scikit-image', 'scikit-image', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/scikit_learn.rb b/lib/docs/scrapers/scikit_learn.rb index 0d7fc90d..973c9d7e 100644 --- a/lib/docs/scrapers/scikit_learn.rb +++ b/lib/docs/scrapers/scikit_learn.rb @@ -25,11 +25,9 @@ module Docs Licensed under the 3-clause BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('https://scikit-learn.org/stable/documentation.html', options) do |doc| - label = doc.at_css('.body h1').content - block.call label.scan(/([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://scikit-learn.org/stable/documentation.html', opts) + doc.at_css('.body h1').content.scan(/([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/sinon.rb b/lib/docs/scrapers/sinon.rb index a1bd06cc..632348dc 100644 --- a/lib/docs/scrapers/sinon.rb +++ b/lib/docs/scrapers/sinon.rb @@ -53,10 +53,9 @@ module Docs self.base_url = "https://sinonjs.org/releases/v#{release}/" end - def get_latest_version(options, &block) - fetch('https://sinonjs.org/', options) do |body| - block.call body.scan(/\/releases\/v([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + body = fetch('https://sinonjs.org/', opts) + body.scan(/\/releases\/v([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/socketio.rb b/lib/docs/scrapers/socketio.rb index 8518ff53..e9bc264a 100644 --- a/lib/docs/scrapers/socketio.rb +++ b/lib/docs/scrapers/socketio.rb @@ -21,8 +21,8 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - get_npm_version('socket.io', options, &block) + def get_latest_version(opts) + get_npm_version('socket.io', opts) end end end diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb index 27720111..d3a5fa6b 100644 --- a/lib/docs/scrapers/sqlite.rb +++ b/lib/docs/scrapers/sqlite.rb @@ -41,10 +41,9 @@ module Docs options[:attribution] = 'SQLite is in the Public Domain.' - def get_latest_version(options, &block) - fetch_doc('https://sqlite.org/chronology.html', options) do |doc| - block.call doc.at_css('#chrontab > tbody > tr > td:last-child > a').content - end + def get_latest_version(opts) + doc = fetch_doc('https://sqlite.org/chronology.html', opts) + doc.at_css('#chrontab > tbody > tr > td:last-child > a').content end private diff --git a/lib/docs/scrapers/statsmodels.rb b/lib/docs/scrapers/statsmodels.rb index 0c9bfd06..db2eacb2 100644 --- a/lib/docs/scrapers/statsmodels.rb +++ b/lib/docs/scrapers/statsmodels.rb @@ -21,10 +21,9 @@ module Docs Licensed under the 3-clause BSD License. HTML - def get_latest_version(options, &block) - fetch_doc('http://www.statsmodels.org/stable/', options) do |doc| - block.call doc.at_css('.sphinxsidebarwrapper h3 + p > b').content - end + def get_latest_version(opts) + doc = fetch_doc('http://www.statsmodels.org/stable/', opts) + doc.at_css('.sphinxsidebarwrapper h3 + p > b').content end end end diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb index 90cd7f4f..d04072da 100644 --- a/lib/docs/scrapers/support_tables.rb +++ b/lib/docs/scrapers/support_tables.rb @@ -179,11 +179,10 @@ module Docs HTML - def get_latest_version(options, &block) - fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', options) do |body| - timestamp = body.scan(/([^<]+)<\/updated>/)[0][0] - block.call DateTime.parse(timestamp).to_time.to_i - end + def get_latest_version(opts) + body = fetch('https://feeds.feedburner.com/WhenCanIUse?format=xml', opts) + timestamp = body.scan(/([^<]+)<\/updated>/)[0][0] + DateTime.parse(timestamp).to_time.to_i end end end diff --git a/lib/docs/scrapers/symfony.rb b/lib/docs/scrapers/symfony.rb index acd105da..439456f0 100644 --- a/lib/docs/scrapers/symfony.rb +++ b/lib/docs/scrapers/symfony.rb @@ -71,10 +71,8 @@ module Docs self.base_url = "https://api.symfony.com/#{version}/" end - def get_latest_version(options, &block) - get_latest_github_release('symfony', 'symfony', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('symfony', 'symfony', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/tcl_tk.rb b/lib/docs/scrapers/tcl_tk.rb index 3dcc5765..bca840c6 100644 --- a/lib/docs/scrapers/tcl_tk.rb +++ b/lib/docs/scrapers/tcl_tk.rb @@ -26,11 +26,9 @@ module Docs Licensed under Tcl/Tk terms HTML - def get_latest_version(options, &block) - fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', options) do |doc| - label = doc.at_css('h2').content - block.call label.scan(/Tk([0-9.]+)/)[0][0] - end + def get_latest_version(opts) + doc = fetch_doc('https://www.tcl.tk/man/tcl/contents.htm', opts) + doc.at_css('h2').content.scan(/Tk([0-9.]+)/)[0][0] end end end diff --git a/lib/docs/scrapers/tensorflow.rb b/lib/docs/scrapers/tensorflow.rb index dfd89875..8ac1ca65 100644 --- a/lib/docs/scrapers/tensorflow.rb +++ b/lib/docs/scrapers/tensorflow.rb @@ -56,10 +56,8 @@ module Docs /\Aextend/] end - def get_latest_version(options, &block) - get_latest_github_release('tensorflow', 'tensorflow', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('tensorflow', 'tensorflow', opts)['tag_name'][1..-1] end private diff --git a/lib/docs/scrapers/terraform.rb b/lib/docs/scrapers/terraform.rb index 73d47e86..0965ad06 100644 --- a/lib/docs/scrapers/terraform.rb +++ b/lib/docs/scrapers/terraform.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MPL 2.0 License. HTML - def get_latest_version(options, &block) - get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', options) do |contents| - block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', opts) + contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] end end end diff --git a/lib/docs/scrapers/twig.rb b/lib/docs/scrapers/twig.rb index 58c7e2ef..e52ff6bd 100755 --- a/lib/docs/scrapers/twig.rb +++ b/lib/docs/scrapers/twig.rb @@ -29,10 +29,9 @@ module Docs self.base_url = 'https://twig.symfony.com/doc/1.x/' end - def get_latest_version(options, &block) - get_github_tags('twigphp', 'Twig', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('twigphp', 'Twig', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb index 9e62d6a9..63d1c9af 100644 --- a/lib/docs/scrapers/typescript.rb +++ b/lib/docs/scrapers/typescript.rb @@ -25,10 +25,8 @@ module Docs Licensed under the Apache License, Version 2.0. HTML - def get_latest_version(options, &block) - get_latest_github_release('Microsoft', 'TypeScript', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('Microsoft', 'TypeScript', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/underscore.rb b/lib/docs/scrapers/underscore.rb index 127dd5d3..a09b4acb 100644 --- a/lib/docs/scrapers/underscore.rb +++ b/lib/docs/scrapers/underscore.rb @@ -21,10 +21,9 @@ module Docs Licensed under the MIT License. HTML - def get_latest_version(options, &block) - fetch_doc('https://underscorejs.org/', options) do |doc| - block.call doc.at_css('.version').content[1...-1] - end + def get_latest_version(opts) + doc = fetch_doc('https://underscorejs.org/', opts) + doc.at_css('.version').content[1...-1] end end end diff --git a/lib/docs/scrapers/vagrant.rb b/lib/docs/scrapers/vagrant.rb index 0e89dad1..7400fc8c 100644 --- a/lib/docs/scrapers/vagrant.rb +++ b/lib/docs/scrapers/vagrant.rb @@ -19,10 +19,9 @@ module Docs Licensed under the MPL 2.0 License. HTML - def get_latest_version(options, &block) - get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', options) do |contents| - block.call contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] - end + def get_latest_version(opts) + contents = get_github_file_contents('hashicorp', 'vagrant', 'website/config.rb', opts) + contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0] end end end diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb index cbb6b15e..db48b898 100644 --- a/lib/docs/scrapers/vue.rb +++ b/lib/docs/scrapers/vue.rb @@ -33,10 +33,8 @@ module Docs self.initial_paths = %w(/api/index.html) end - def get_latest_version(options, &block) - get_latest_github_release('vuejs', 'vue', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('vuejs', 'vue', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/vulkan.rb b/lib/docs/scrapers/vulkan.rb index 111e405f..da5d696d 100644 --- a/lib/docs/scrapers/vulkan.rb +++ b/lib/docs/scrapers/vulkan.rb @@ -21,10 +21,9 @@ module Docs Vulkan and the Vulkan logo are registered trademarks of the Khronos Group Inc. HTML - def get_latest_version(options, &block) - get_github_tags('KhronosGroup', 'Vulkan-Docs', options) do |tags| - block.call tags[0]['name'][1..-1] - end + def get_latest_version(opts) + tags = get_github_tags('KhronosGroup', 'Vulkan-Docs', opts) + tags[0]['name'][1..-1] end end end diff --git a/lib/docs/scrapers/webpack.rb b/lib/docs/scrapers/webpack.rb index 9d7303ed..7af2cf11 100644 --- a/lib/docs/scrapers/webpack.rb +++ b/lib/docs/scrapers/webpack.rb @@ -69,8 +69,8 @@ module Docs HTML end - def get_latest_version(options, &block) - get_npm_version('webpack', options, &block) + def get_latest_version(opts) + get_npm_version('webpack', opts) end end end diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb index 26f7c0cc..d63e7121 100644 --- a/lib/docs/scrapers/yarn.rb +++ b/lib/docs/scrapers/yarn.rb @@ -21,10 +21,8 @@ module Docs Licensed under the BSD License. HTML - def get_latest_version(options, &block) - get_latest_github_release('yarnpkg', 'yarn', options) do |release| - block.call release['tag_name'][1..-1] - end + def get_latest_version(opts) + get_latest_github_release('yarnpkg', 'yarn', opts)['tag_name'][1..-1] end end end diff --git a/lib/docs/scrapers/yii.rb b/lib/docs/scrapers/yii.rb index 0d087abb..b05c5bf7 100755 --- a/lib/docs/scrapers/yii.rb +++ b/lib/docs/scrapers/yii.rb @@ -35,10 +35,8 @@ module Docs options[:container] = '.grid_9' end - def get_latest_version(options, &block) - get_latest_github_release('yiisoft', 'yii2', options) do |release| - block.call release['tag_name'] - end + def get_latest_version(opts) + get_latest_github_release('yiisoft', 'yii2', opts)['tag_name'] end end end diff --git a/lib/tasks/updates.thor b/lib/tasks/updates.thor index e684cdbc..b7e251fb 100644 --- a/lib/tasks/updates.thor +++ b/lib/tasks/updates.thor @@ -62,17 +62,15 @@ class UpdatesCLI < Thor logger.debug("Checking #{doc.name}") instance = doc.versions.first.new + scraper_version = instance.get_scraper_version(opts) + latest_version = instance.get_latest_version(opts) - instance.get_scraper_version(opts) do |scraper_version| - instance.get_latest_version(opts) do |latest_version| - return { - name: doc.name, - scraper_version: format_version(scraper_version), - latest_version: format_version(latest_version), - is_outdated: instance.is_outdated(scraper_version, latest_version) - } - end - end + { + name: doc.name, + scraper_version: format_version(scraper_version), + latest_version: format_version(latest_version), + is_outdated: instance.is_outdated(scraper_version, latest_version) + } rescue NotImplementedError logger.warn("Couldn't check #{doc.name}, get_latest_version is not implemented") error_result(doc, '`get_latest_version` is not implemented') @@ -87,7 +85,7 @@ class UpdatesCLI < Thor # If the version is numeric and greater than or equal to 1e9 it's probably a timestamp return str if str.match(/^(\d)+$/).nil? or str.to_i < 1e9 - DateTime.strptime(str, '%s').strftime('%B %-d, %Y') + DateTime.strptime(str, '%s').strftime('%F') end def error_result(doc, reason) @@ -150,46 +148,45 @@ class UpdatesCLI < Thor logger.info('Uploading the results to a new GitHub issue') logger.info('Checking if the GitHub token belongs to the correct user') - github_get('/user') do |user| - # Only allow the DevDocs bot to upload reports - if user['login'] == UPLOAD_USER - issue = results_to_issue(outdated_results, up_to_date_results, failed_results) - - logger.info('Creating a new GitHub issue') - github_post("/repos/#{UPLOAD_REPO}/issues", issue) do |created_issue| - search_params = { - q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", - sort: 'created', - order: 'desc' - } - - logger.info('Checking if the previous issue is still open') - github_get('/search/issues', search_params) do |matching_issues| - previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} - - if previous_issue.nil? - logger.info('No previous issue found') - log_upload_success(created_issue) - else - comment = "This report was superseded by ##{created_issue['number']}." - - logger.info('Commenting on the previous issue') - github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) do |_| - if previous_issue['closed_at'].nil? - logger.info('Closing the previous issue') - github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) do |_| - log_upload_success(created_issue) - end - else - logger.info('The previous issue has already been closed') - log_upload_success(created_issue) - end - end - end - end - end + user = github_get('/user') + + # Only allow the DevDocs bot to upload reports + unless user['login'] == UPLOAD_USER + logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + return + end + + logger.info('Creating a new GitHub issue') + + issue = results_to_issue(outdated_results, up_to_date_results, failed_results) + created_issue = github_post("/repos/#{UPLOAD_REPO}/issues", issue) + + logger.info('Checking if the previous issue is still open') + + search_params = { + q: "Documentation versions report in:title author:#{UPLOAD_USER} is:issue repo:#{UPLOAD_REPO}", + sort: 'created', + order: 'desc' + } + + matching_issues = github_get('/search/issues', search_params) + previous_issue = matching_issues['items'].find {|item| item['number'] != created_issue['number']} + + if previous_issue.nil? + logger.info('No previous issue found') + log_upload_success(created_issue) + else + logger.info('Commenting on the previous issue') + + comment = "This report was superseded by ##{created_issue['number']}." + github_post("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}/comments", {body: comment}) + if previous_issue['closed_at'].nil? + logger.info('Closing the previous issue') + github_patch("/repos/#{UPLOAD_REPO}/issues/#{previous_issue['number']}", {state: 'closed'}) + log_upload_success(created_issue) else - logger.error("Only #{UPLOAD_USER} is supposed to upload the results to a new issue. The specified github token is not for #{UPLOAD_USER}.") + logger.info('The previous issue has already been closed') + log_upload_success(created_issue) end end end @@ -266,19 +263,19 @@ The #{outdated_results.length + up_to_date_results.length + failed_results.lengt # HTTP utilities # - def github_get(endpoint, params = {}, &block) - github_request(endpoint, {method: :get, params: params}, &block) + def github_get(endpoint, params = {}) + github_request(endpoint, {method: :get, params: params}) end - def github_post(endpoint, params, &block) - github_request(endpoint, {method: :post, body: params.to_json}, &block) + def github_post(endpoint, params) + github_request(endpoint, {method: :post, body: params.to_json}) end - def github_patch(endpoint, params, &block) - github_request(endpoint, {method: :patch, body: params.to_json}, &block) + def github_patch(endpoint, params) + github_request(endpoint, {method: :patch, body: params.to_json}) end - def github_request(endpoint, opts, &block) + def github_request(endpoint, opts) url = "https://api.github.com#{endpoint}" # GitHub token authentication @@ -292,16 +289,15 @@ The #{outdated_results.length + up_to_date_results.length + failed_results.lengt end logger.debug("Making a #{opts[:method]} request to #{url}") - - Docs::Request.run(url, opts) do |response| - # response.success? is false if the response code is 201 - # GitHub returns 201 Created after an issue is created - if response.success? || response.code == 201 - block.call JSON.parse(response.body) - else - logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") - block.call nil - end + response = Docs::Request.run(url, opts) + + # response.success? is false if the response code is 201 + # GitHub returns 201 Created after an issue is created + if response.success? || response.code == 201 + JSON.parse(response.body) + else + logger.error("Couldn't make a #{opts[:method]} request to #{url} (response code #{response.code})") + nil end end