public/icons/*your_scraper_name*/ are up-to-date if the documentation has a custom icon
- [ ] Ensured `self.links` contains up-to-date urls if `self.links` is defined
- [ ] Tested the changes locally to ensure:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4e6c95e0..ad547735 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -11,9 +11,9 @@ jobs:
runs-on: ubuntu-20.04
if: github.repository == 'freeCodeCamp/devdocs'
steps:
- - uses: actions/checkout@v2.3.4
+ - uses: actions/checkout@v2.4.2
- name: Set up Ruby
- uses: ruby/setup-ruby@v1.66.1
+ uses: ruby/setup-ruby@v1.110.0
with:
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: Run tests
diff --git a/.github/workflows/schedule-doc-report.yml b/.github/workflows/schedule-doc-report.yml
index 013ed75b..b90a3098 100644
--- a/.github/workflows/schedule-doc-report.yml
+++ b/.github/workflows/schedule-doc-report.yml
@@ -8,9 +8,9 @@ jobs:
runs-on: ubuntu-20.04
if: github.repository == 'freeCodeCamp/devdocs'
steps:
- - uses: actions/checkout@v2.3.4
+ - uses: actions/checkout@v2.4.2
- name: Set up Ruby
- uses: ruby/setup-ruby@v1.66.1
+ uses: ruby/setup-ruby@v1.110.0
with:
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- name: Generate report
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 00000000..7b7801dc
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,18 @@
+name: Ruby tests
+
+on:
+ pull_request:
+ branches:
+ - main
+
+jobs:
+ test:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v2.4.2
+ - name: Set up Ruby
+ uses: ruby/setup-ruby@v1.110.0
+ with:
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+ - name: Run tests
+ run: bundle exec rake
diff --git a/.gitignore b/.gitignore
index 8f06d308..bbf749a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,7 @@ public/docs/**/*
docs/**/*
!docs/*.md
/vendor
+*.tar
+*.tar.bz2
+*.tar.gz
+*.zip
diff --git a/.ruby-version b/.ruby-version
index 57cf282e..49cdd668 100644
--- a/.ruby-version
+++ b/.ruby-version
@@ -1 +1 @@
-2.6.5
+2.7.6
diff --git a/Dockerfile b/Dockerfile
index dd1302df..060193df 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM ruby:2.6.5
+FROM ruby:2.7.6
ENV LANG=C.UTF-8
ENV ENABLE_SERVICE_WORKER=true
diff --git a/Gemfile b/Gemfile
index 61004330..fa6189fd 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,9 +1,9 @@
source 'https://rubygems.org'
-ruby '~> 2.6.0'
+ruby '~> 2.7.3'
gem 'rake'
gem 'thor'
-gem 'pry', '~> 0.12.0'
+gem 'pry', '~> 0.14.0'
gem 'activesupport', '~> 5.2', require: false
gem 'yajl-ruby', require: false
gem 'html-pipeline'
diff --git a/Gemfile.lock b/Gemfile.lock
index a73db036..f91f2685 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,38 +1,38 @@
GEM
remote: https://rubygems.org/
specs:
- activesupport (5.2.3)
+ activesupport (5.2.6)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
tzinfo (~> 1.1)
backports (3.15.0)
- better_errors (2.5.1)
+ better_errors (2.9.1)
coderay (>= 1.0.0)
erubi (>= 1.0.0)
rack (>= 0.9.0)
browser (2.6.1)
chunky_png (1.3.11)
- coderay (1.1.2)
+ coderay (1.1.3)
coffee-script (2.4.1)
coffee-script-source
execjs
coffee-script-source (1.12.2)
- concurrent-ruby (1.1.5)
- daemons (1.3.1)
- erubi (1.9.0)
+ concurrent-ruby (1.1.9)
+ daemons (1.4.0)
+ erubi (1.10.0)
ethon (0.12.0)
ffi (>= 1.3.0)
eventmachine (1.2.7)
execjs (2.7.0)
exifr (1.3.6)
- ffi (1.12.2)
+ ffi (1.15.5)
fspath (3.1.2)
highline (2.0.3)
html-pipeline (2.12.0)
activesupport (>= 2)
nokogiri (>= 1.4)
- i18n (1.7.0)
+ i18n (1.8.10)
concurrent-ruby (~> 1.0)
image_optim (0.26.5)
exifr (~> 1.2, >= 1.2.2)
@@ -45,26 +45,29 @@ GEM
image_optim (~> 0.19)
image_size (2.0.2)
in_threads (1.5.3)
- method_source (0.9.2)
- mini_portile2 (2.4.0)
- minitest (5.12.2)
+ method_source (1.0.0)
+ mini_portile2 (2.8.0)
+ minitest (5.14.4)
multi_json (1.13.1)
- mustermann (1.0.3)
+ mustermann (1.1.1)
+ ruby2_keywords (~> 0.0.1)
net-sftp (3.0.0.beta1)
net-ssh (>= 5.0.0, < 6.0.0)
net-ssh (5.2.0)
newrelic_rpm (6.7.0.359)
- nokogiri (1.10.4)
- mini_portile2 (~> 2.4.0)
+ nokogiri (1.13.6)
+ mini_portile2 (~> 2.8.0)
+ racc (~> 1.4)
options (2.3.2)
progress (3.5.2)
progress_bar (1.3.0)
highline (>= 1.6, < 3)
options (~> 2.3.0)
- pry (0.12.2)
- coderay (~> 1.1.0)
- method_source (~> 0.9.0)
- rack (2.0.7)
+ pry (0.14.1)
+ coderay (~> 1.1)
+ method_source (~> 1.0)
+ racc (1.6.0)
+ rack (2.2.3.1)
rack-protection (2.0.7)
rack
rack-ssl-enforcer (0.2.9)
@@ -75,6 +78,7 @@ GEM
rb-inotify (0.10.0)
ffi (~> 1.0)
rr (1.2.1)
+ ruby2_keywords (0.0.5)
sass (3.7.4)
sass-listen (~> 4.0.0)
sass-listen (4.0.0)
@@ -106,7 +110,7 @@ GEM
strings-ansi (0.1.0)
terminal-table (1.8.0)
unicode-display_width (~> 1.1, >= 1.1.1)
- thin (1.7.2)
+ thin (1.8.1)
daemons (~> 1.0, >= 1.0.9)
eventmachine (~> 1.0, >= 1.0.4)
rack (>= 1, < 3)
@@ -121,14 +125,14 @@ GEM
tty-which (0.4.1)
typhoeus (1.3.1)
ethon (>= 0.9.0)
- tzinfo (1.2.5)
+ tzinfo (1.2.9)
thread_safe (~> 0.1)
uglifier (4.2.0)
execjs (>= 0.3.0, < 3)
unicode-display_width (1.6.0)
unicode_utils (1.4.0)
unix_utils (0.0.15)
- yajl-ruby (1.4.1)
+ yajl-ruby (1.4.3)
PLATFORMS
ruby
@@ -148,7 +152,7 @@ DEPENDENCIES
newrelic_rpm
nokogiri
progress_bar
- pry (~> 0.12.0)
+ pry (~> 0.14.0)
rack
rack-ssl-enforcer
rack-test
@@ -170,7 +174,7 @@ DEPENDENCIES
yajl-ruby
RUBY VERSION
- ruby 2.6.3p62
+ ruby 2.7.3p183
BUNDLED WITH
- 2.0.2
+ 2.1.4
diff --git a/README.md b/README.md
index 35367313..f7788a8b 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Unless you wish to contribute to the project, we recommend using the hosted vers
DevDocs is made of two pieces: a Ruby scraper that generates the documentation and metadata, and a JavaScript app powered by a small Sinatra app.
-DevDocs requires Ruby 2.6.x, libcurl, and a JavaScript runtime supported by [ExecJS](https://github.com/rails/execjs#readme) (included in OS X and Windows; [Node.js](https://nodejs.org/en/) on Linux). Once you have these installed, run the following commands:
+DevDocs requires Ruby 2.7.4, libcurl, and a JavaScript runtime supported by [ExecJS](https://github.com/rails/execjs#readme) (included in OS X and Windows; [Node.js](https://nodejs.org/en/) on Linux). Once you have these installed, run the following commands:
```sh
git clone https://github.com/freeCodeCamp/devdocs.git && cd devdocs
@@ -147,8 +147,6 @@ Made something cool? Feel free to open a PR to add a new row to this table!
| Project | Description | Last commit |
|---------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Chrome web app](https://chrome.google.com/webstore/detail/devdocs/mnfehgbmkapmjnhcnbodoamcioleeooe) | Chrome Web App which adds a shortcut to DevDocs apps page. | N/A |
-| [Ubuntu Touch app](https://uappexplorer.com/app/devdocsunofficial.berkes) | Application for devices running Ubuntu Touch. | N/A |
| [Sublime Text plugin](https://sublime.wbond.net/packages/DevDocs) | Sublime Text plugin to search DevDocs by selection or by input. | [](https://github.com/vitorbritto/sublime-devdocs) |
| [Atom plugin](https://atom.io/packages/devdocs) | Atom plugin adding the `doc` command to search DevDocs. | [](https://github.com/masnun/atom-devdocs) |
| [gruehle/dev-docs-viewer](https://github.com/gruehle/dev-docs-viewer) | Brackets extension for searching and viewing DevDocs content. | [](https://github.com/gruehle/dev-docs-viewer) |
@@ -163,6 +161,8 @@ Made something cool? Feel free to open a PR to add a new row to this table!
| [hardpixel/devdocs-desktop](https://github.com/hardpixel/devdocs-desktop) | GTK3 application for DevDocs with search integrated in the headerbar. | [](https://github.com/hardpixel/devdocs-desktop) |
| [dteoh/devdocs-macos](https://github.com/dteoh/devdocs-macos) | Native macOS application for DevDocs. | [](https://github.com/dteoh/devdocs-macos) |
| [Merith-TK/devdocs_webapp_kotlin](https://github.com/Merith-TK/devdocs_webapp_kotlin) | Android application which shows DevDocs in a webview. | [](https://github.com/Merith-TK/devdocs_webapp_kotlin) |
+| [astoff/devdocs.el](https://github.com/astoff/devdocs.el) | Emacs viewer for DevDocs | [](https://github.com/astoff/devdocs.el) |
+| [DevDocs Tab for VS Code](https://github.com/mohamed3nan/DevDocs-Tab) | VS Code extension to search for documentation on DevDocs.io faster by displaying it in a tab inside VS Code.| [](https://github.com/mohamed3nan/DevDocs-Tab) |
## Copyright / License
diff --git a/assets/javascripts/app/settings.coffee b/assets/javascripts/app/settings.coffee
index d3fc93e1..38b4f910 100644
--- a/assets/javascripts/app/settings.coffee
+++ b/assets/javascripts/app/settings.coffee
@@ -13,6 +13,8 @@ class app.Settings
'size'
'tips'
'autoInstall'
+ 'spaceScroll'
+ 'spaceTimeout'
]
INTERNAL_KEYS = [
@@ -22,7 +24,12 @@ class app.Settings
'news'
]
- LAYOUTS: ['_max-width', '_sidebar-hidden', '_native-scrollbars']
+ LAYOUTS: [
+ '_max-width'
+ '_sidebar-hidden'
+ '_native-scrollbars'
+ '_text-justify-hyphenate'
+ ]
@defaults:
count: 0
@@ -33,6 +40,8 @@ class app.Settings
schema: 1
analyticsConsent: false
theme: 'auto'
+ spaceScroll: 1
+ spaceTimeout: 0.5
constructor: ->
@store = new CookiesStore
@@ -149,7 +158,8 @@ class app.Settings
toggleLayout: (layout, enable) ->
classList = document.body.classList
- classList.toggle(layout, enable) unless app.router?.isSettings
+ # sidebar is always shown for settings; its state is updated in app.views.Settings
+ classList.toggle(layout, enable) unless layout is '_sidebar-hidden' and app.router?.isSettings
classList.toggle('_overlay-scrollbars', $.overlayScrollbarsEnabled())
return
diff --git a/assets/javascripts/app/shortcuts.coffee b/assets/javascripts/app/shortcuts.coffee
index fb0c7220..28ddf0b8 100644
--- a/assets/javascripts/app/shortcuts.coffee
+++ b/assets/javascripts/app/shortcuts.coffee
@@ -18,10 +18,16 @@ class app.Shortcuts
swapArrowKeysBehavior: ->
app.settings.get('arrowScroll')
+ spaceScroll: ->
+ app.settings.get('spaceScroll')
+
showTip: ->
app.showTip('KeyNav')
@showTip = null
+ spaceTimeout: ->
+ app.settings.get('spaceTimeout')
+
onKeydown: (event) =>
return if @buggyEvent(event)
result = if event.ctrlKey or event.metaKey
@@ -59,7 +65,7 @@ class app.Shortcuts
@trigger 'escape'
false
when 32
- if event.target.type is 'search' and (not @lastKeypress or @lastKeypress < Date.now() - 500)
+ if event.target.type is 'search' and @spaceScroll() and (not @lastKeypress or @lastKeypress < Date.now() - (@spaceTimeout() * 1000))
@trigger 'pageDown'
false
when 33
diff --git a/assets/javascripts/lib/page.coffee b/assets/javascripts/lib/page.coffee
index 7a5329f6..5ad89b32 100644
--- a/assets/javascripts/lib/page.coffee
+++ b/assets/javascripts/lib/page.coffee
@@ -200,6 +200,8 @@ page.track = (fn) ->
track = ->
return unless app.config.env == 'production'
+ return if navigator.doNotTrack == '1'
+ return if navigator.globalPrivacyControl
consentGiven = Cookies.get('analyticsConsent')
consentAsked = Cookies.get('analyticsConsentAsked')
diff --git a/assets/javascripts/news.json b/assets/javascripts/news.json
index 1757b39d..2a9934a9 100644
--- a/assets/javascripts/news.json
+++ b/assets/javascripts/news.json
@@ -1,4 +1,61 @@
[
+ [
+ "2022-05-03",
+ "New documentations: Kubernetes, Kubectl"
+ ],
+ [
+ "2022-04-25",
+ "New documentation: Nix"
+ ],
+ [
+ "2022-03-31",
+ "New documentation: Eigen3"
+ ],
+ [
+ "2022-02-21",
+ "New documentation: Tailwind CSS"
+ ],
+ [
+ "2022-01-12",
+ "New documentation: React Router"
+ ],
+ [
+ "2022-01-09",
+ "New documentation: Deno"
+ ],
+ [
+ "2021-12-29",
+ "New documentation: PointCloudLibrary"
+ ],
+ [
+ "2021-12-27",
+ "New documentation: Zig"
+ ],
+ [
+ "2021-12-26",
+ "New documentation: GNU Make"
+ ],
+ [
+ "2021-12-07",
+ "New documentation: Prettier",
+ "Renamed documentation: Web APIs"
+ ],
+ [
+ "2021-12-05",
+ "New documentation: esbuild"
+ ],
+ [
+ "2021-12-04",
+ "New documentation: Vite"
+ ],
+ [
+ "2021-11-29",
+ "New documentation: i3"
+ ],
+ [
+ "2021-06-09",
+ "New documentation: R"
+ ],
[
"2021-05-31",
"New documentation: Web Extensions"
diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee
index 1698eec2..ebadb1b3 100644
--- a/assets/javascripts/templates/pages/about_tmpl.coffee
+++ b/assets/javascripts/templates/pages/about_tmpl.coffee
@@ -30,7 +30,6 @@ app.templates.aboutPage = -> """
')
+ prefix_with(title, 'pkgs')
+ end
+ end
+ new_root.add_child(special_sections)
+
+ # image builders
+ image_sections = xpath("//*[@id='chap-images']/ancestor::section[1]/section")
+ image_sections.css('.titlepage .title').each do |title|
+ strip_section_number(title.children)
+ title.children[0].wrap('')
+ end
+ image_sections.each do |section|
+ prefix = section.at_xpath('*[@class="titlepage"]//*[@class="title"]').content
+ next unless ["pkgs.dockerTools", "pkgs.ociTools"].include?(prefix)
+ section.xpath('section/*[@class="titlepage"]//*[@class="title"]').each do |title|
+ prefix_with(title, prefix)
+ title['data-add-to-index'] = ''
+ end
+ end
+ new_root.add_child(image_sections)
+
+ new_root.css('pre.programlisting').attr('data-language', 'nix')
+
+ new_root
+ elsif subpath == 'nix/stable/expressions/builtins.html'
+ @doc = doc.at_css('main dl')
+
+ # strip out the first entry, `derivation`, the actual documentation
+ # exists in a separate page
+ derivation_dt = doc.children.at_css('dt')
+ if derivation_dt.content.starts_with?('derivation')
+ derivation_dt.remove
+ doc.children.at_css('dd').remove
+ else
+ raise RuntimeError.new('First entry is not derivation, update the scraper')
+ end
+
+ doc.css('dt').each do |title|
+ title.name = 'h2'
+ unwrap(title.at_css('a'))
+ title.children[0].children.before('builtins.')
+ end
+ doc.css('dd').each do |description|
+ description.name = 'div'
+ end
+
+ doc.css('pre > code').each do |code|
+ code.parent['data-language'] = 'nix' if code['class'] == 'language-nix'
+ code.parent['data-language'] = 'xml' if code['class'] == 'language-xml'
+ unwrap(code)
+ end
+
+ doc
+ else
+ doc
+ end
+ end
+
+ def strip_section_number(title_children)
+ while title_children.first.content == ''
+ title_children.shift.remove
+ end
+ first_text = title_children.first
+ return unless first_text.text?
+ first_text.content = first_text.content.sub(/[0-9.]+ ?/, '')
+ first_text.remove if first_text.blank?
+ end
+
+ def prefix_with(title_node, text)
+ title_node.css('code').each do |code|
+ code.content = "#{text}.#{code.content}" unless code.content.starts_with?("#{text}.")
+ end
+ end
+
+ def unwrap(node)
+ node.replace(node.inner_html)
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/nix/entries.rb b/lib/docs/filters/nix/entries.rb
new file mode 100644
index 00000000..f4ff093a
--- /dev/null
+++ b/lib/docs/filters/nix/entries.rb
@@ -0,0 +1,22 @@
+module Docs
+ class Nix
+ class EntriesFilter < Docs::EntriesFilter
+ def include_default_entry?
+ false
+ end
+
+ def additional_entries
+ css('h2, h3[data-add-to-index]').flat_map do |node|
+ node.css('code').map do |code|
+ title = code.content
+ index = title.rindex('.')
+ type = title[0...index]
+ name = title.match(/^[^\s]+/)[0]
+
+ [name, node['id'], type]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/npm/clean_html.rb b/lib/docs/filters/npm/clean_html.rb
index 3f85bf6c..ad07c476 100644
--- a/lib/docs/filters/npm/clean_html.rb
+++ b/lib/docs/filters/npm/clean_html.rb
@@ -3,6 +3,10 @@ module Docs
class CleanHtmlFilter < Filter
def call
+ at_css('#___gatsby').before(at_css('h1'))
+
+ css('details').remove
+
css('.dZYhXG', '.fONtKn').remove
css('.kSYjyK').remove
@@ -11,6 +15,11 @@ module Docs
css('.jRndWL').remove_attribute('style')
+ css('pre').each do |node|
+ node.content = node.css('.token-line').map(&:content).join("\n")
+ node['data-language'] = 'javascript'
+ end
+
doc
end
diff --git a/lib/docs/filters/numpy/clean_html.rb b/lib/docs/filters/numpy/clean_html.rb
index 54871eed..33085662 100644
--- a/lib/docs/filters/numpy/clean_html.rb
+++ b/lib/docs/filters/numpy/clean_html.rb
@@ -2,7 +2,8 @@ module Docs
class Numpy
class CleanHtmlFilter < Filter
def call
- at_css('#spc-section-body, main > div')
+ css('.sphinx-bs.container.pb-4.docutils').remove if root_page?
+ at_css('main > div > section', '#spc-section-body, main > div')
end
end
end
diff --git a/lib/docs/filters/numpy/entries.rb b/lib/docs/filters/numpy/entries.rb
index e6fbb7d1..f6c43dd7 100644
--- a/lib/docs/filters/numpy/entries.rb
+++ b/lib/docs/filters/numpy/entries.rb
@@ -19,7 +19,7 @@ module Docs
return 'Development'
end
li_a = css('nav li.active > a')
- return li_a.last.content if li_a
+ return li_a.last.content if li_a && li_a.last
end
nav_items = css('.nav.nav-pills.pull-left > li')
@@ -71,6 +71,7 @@ module Docs
def additional_entries
css('dl:not(:first-of-type) > dt[id]').each_with_object [] do |node, entries|
+ next if node.ancestors('.citation').present?
name = dt_to_name(node)
if type == 'NumPy C API'
diff --git a/lib/docs/filters/octave/entries.rb b/lib/docs/filters/octave/entries.rb
index 2e5c0de6..ea4c37f4 100644
--- a/lib/docs/filters/octave/entries.rb
+++ b/lib/docs/filters/octave/entries.rb
@@ -13,7 +13,7 @@ module Docs
def additional_entries
css('dl:not([compact]) > dt').each_with_object [] do |node, entries|
- name = node.content.gsub(/[A-z0-9\,… ]*\=/, '').strip.split(' ')[0]
+ name = node.content.gsub(/^: +/, '').gsub(/[A-z0-9\,… ]*\=/, '').strip.split(' ')[0]
entries << [name, node['id'], 'Functions'] unless node['id'] =~ /-\d+\Z/
end
end
diff --git a/lib/docs/filters/openjdk/clean_html.rb b/lib/docs/filters/openjdk/clean_html.rb
index 4cd6afe0..5dbbd771 100644
--- a/lib/docs/filters/openjdk/clean_html.rb
+++ b/lib/docs/filters/openjdk/clean_html.rb
@@ -111,6 +111,7 @@ module Docs
css('hr + br', 'p + br', 'div + br', 'hr').remove
css('pre').each do |node|
+ node.content = node.content.sub(/\u200B/, '') # fix zero width space characters
node.content = node.content.strip
node['data-language'] = 'java'
end
diff --git a/lib/docs/filters/openjdk/clean_html_new.rb b/lib/docs/filters/openjdk/clean_html_new.rb
index 1b6d4816..90341926 100644
--- a/lib/docs/filters/openjdk/clean_html_new.rb
+++ b/lib/docs/filters/openjdk/clean_html_new.rb
@@ -7,6 +7,10 @@ module Docs
at_css('h1').content = "OpenJDK #{release} Documentation"
end
+ css('.header > h1').each do |node|
+ node.parent.before(node).remove
+ end
+
css('.header .sub-title', 'hr', '.table-tabs').remove
# fix ul section that contains summaries or tables
@@ -30,7 +34,9 @@ module Docs
end
# add syntax highlight to each method
- css('.member-signature').each do |node|
+ css('.type-signature, .member-signature').each do |node|
+ node.content = node.content.sub(/\u200B/, '') # fix zero width space characters
+
node.name = 'pre'
node['class'] = 'lang-java'
node['data-language'] = 'java'
@@ -38,7 +44,25 @@ module Docs
node.css('span').each do |subnode|
subnode.name = 'code'
end
+ end
+ # convert pseudo tables (made from div) to real tables
+ css('div.caption').remove
+ css('.two-column-summary > .col-constructor-name').add_class('col-first')
+ css('.two-column-summary, .three-column-summary').each do |table|
+ # table.previous_element.remove if table.previous_element?.classes?.include?('caption')
+ table.name = 'table'
+ tr = nil
+ table.css('div.col-first, div.col-second, div.col-last').each do |td|
+ if td.classes.include?('col-first')
+ table.add_child('')
+ tr = table.last_element_child
+ end
+ td.name = 'td'
+ td.name = 'th' if td.classes.include?('table-header')
+ td.remove_attribute('class')
+ tr.add_child(td.remove)
+ end
end
doc
diff --git a/lib/docs/filters/pandas/clean_html.rb b/lib/docs/filters/pandas/clean_html.rb
index 4b0fd3bf..86900be4 100644
--- a/lib/docs/filters/pandas/clean_html.rb
+++ b/lib/docs/filters/pandas/clean_html.rb
@@ -23,6 +23,9 @@ module Docs
node['data-language'] = 'python'
end
+ # table of contents "on this page"
+ css('.toc-item').remove
+
# sidebar
css('ul.nav.bd-sidenav').remove
@@ -30,8 +33,7 @@ module Docs
css('.headerlink').remove
# next and previous section buttons
- css('next-link').remove
- css('prev-link').remove
+ css('.prev-next-area').remove
css('footer').remove
diff --git a/lib/docs/filters/point_cloud_library/clean_html.rb b/lib/docs/filters/point_cloud_library/clean_html.rb
new file mode 100644
index 00000000..143345d6
--- /dev/null
+++ b/lib/docs/filters/point_cloud_library/clean_html.rb
@@ -0,0 +1,12 @@
+module Docs
+ class PointCloudLibrary
+ class CleanHtmlFilter < Filter
+ def call
+ @doc = at_css('.contents')
+ css('.dynheader.closed').remove
+ css('.permalink').remove
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/point_cloud_library/entries.rb b/lib/docs/filters/point_cloud_library/entries.rb
new file mode 100644
index 00000000..29b22214
--- /dev/null
+++ b/lib/docs/filters/point_cloud_library/entries.rb
@@ -0,0 +1,45 @@
+module Docs
+ class PointCloudLibrary
+ class EntriesFilter < Docs::EntriesFilter
+ def get_type
+ group = at_css('.title .ingroups')
+ return group.content unless group.nil?
+ name = get_name
+ return 'pcl' unless name.match(/^pcl::.*::/)
+ name.gsub(/^pcl::/, '').gsub(/::.*/, '')
+ end
+
+ def get_name
+ at_css('.title').content.gsub(/[<(].*/, '')
+ end
+
+ def additional_entries
+ # Only add additional_entries from PointCloudLibrary modules (group__*.html)
+ return [] if not slug.include?("group")
+ entries = []
+
+ css('table.memberdecls td.memItemRight').map do |node|
+ href = node.at_css("a").attr('href')
+ if href.index("#").nil? then
+ href += "#"
+ end
+
+ # Skip page that's not crawled
+ # TODO: Sync this with options[:skip_patterns] in point_cloud_library.rb
+ if href.include?("namespace") || href.include?("structsvm") || href.include?("classopenni") then
+ next
+ end
+
+ # Only add function and classes documentation
+ doxygen_type = node.parent.parent.at_css("tr.heading").text.strip
+ if not(doxygen_type == "Functions" || doxygen_type == "Classes") then
+ next
+ end
+
+ entries << [node.content, href]
+ end
+ entries
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/postgresql/entries.rb b/lib/docs/filters/postgresql/entries.rb
index 4ddac456..8873a66e 100644
--- a/lib/docs/filters/postgresql/entries.rb
+++ b/lib/docs/filters/postgresql/entries.rb
@@ -100,21 +100,14 @@ module Docs
when 'queries-table-expressions'
entries.concat get_heading_entries('h3[id], .sect3[id] > h3:first-child')
entries.concat get_custom_entries('dt > .literal:first-child')
- when 'functions-logical'
- entries.concat get_custom_entries('> table td:first-child > code')
- when 'functions-formatting'
- entries.concat get_custom_entries('#FUNCTIONS-FORMATTING-TABLE td:first-child > code')
when 'functions-admin'
- entries.concat get_custom_entries('.table td:first-child > code')
- when 'functions-string'
- entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')
- entries.concat get_custom_entries('> div[id^="FORMAT"] td:first-child > code')
+ entries.concat get_custom_entries('.table td:first-child > p:first-child > code.function')
else
if type && type.start_with?('Functions')
entries.concat get_custom_entries('> .table td:first-child > code.literal:first-child')
entries.concat get_custom_entries('> .table td:first-child > code.function:first-child')
entries.concat get_custom_entries('> .table td:first-child > code:not(.literal):first-child + code.literal')
- entries.concat get_custom_entries('> .table td:first-child > p > code.literal:first-child')
+ entries.concat get_custom_entries('> .table td:first-child > p:first-child > code.literal:first-child')
entries.concat get_custom_entries('> .table td:first-child > p > code.function:first-child')
entries.concat get_custom_entries('> .table td:first-child > p > code:not(.literal):first-child + code.literal')
if slug == 'functions-comparison' && !at_css('#FUNCTIONS-COMPARISON-PRED-TABLE') # before 9.6
@@ -220,6 +213,7 @@ module Docs
unless entries.any? { |entry| entry[0] == name }
node['id'] = id
entries << [name, id]
+ # puts [selector, name].join(' --> ')
end
end
end
diff --git a/lib/docs/filters/prettier/clean_html.rb b/lib/docs/filters/prettier/clean_html.rb
new file mode 100644
index 00000000..7af0bd9c
--- /dev/null
+++ b/lib/docs/filters/prettier/clean_html.rb
@@ -0,0 +1,83 @@
+module Docs
+ class Prettier
+ class CleanHtmlFilter < Filter
+ def call
+ @doc = at_css('.post')
+
+ if root_page?
+ at_css('h1').content = 'React Native Documentation'
+ css('h1 ~ *').remove
+ end
+
+ css('.docs-prevnext', '.hash-link', '.edit-page-link', '.edit-github', 'a.hash', '.edit-page-block', 'a.show', 'a.hide', 'hr').remove
+
+ css('table h1', 'table h2', 'table h3').each do |node|
+ table = node
+ table = table.parent until table.name == 'table'
+ table.replace(node)
+ end
+
+ css('a.anchor', 'a.hashref').each do |node|
+ node.parent['id'] ||= node['name'] || node['id']
+ end
+
+ css('.highlight').each do |node|
+ node.name = 'pre'
+ node.css('.gutter').remove
+ node['data-language'] = node.at_css('[data-lang]').try(:[], 'data-lang') || 'js'
+ node.content = node.content.strip
+ end
+
+ css('table.highlighttable').each do |node|
+ node.replace(node.at_css('pre.highlight'))
+ end
+
+ css('.prism').each do |node|
+ node.name = 'pre'
+ node['data-language'] = node['class'][/(?<=language\-)(\w+)/]
+ node.content = node.content
+ end
+
+ css('pre > code.hljs').each do |node|
+ node.parent['data-language'] = 'jsx'
+ node.before(node.children).remove
+ end
+
+ css('blockquote > p:first-child').each do |node|
+ node.remove if node.content.strip == 'Note:'
+ end
+
+ css('h3#props', 'h3#methods').each { |node| node.name = 'h2' }
+ css('h4.propTitle').each { |node| node.name = 'h3' }
+
+ css('> div > div', '> div', 'div > span', '.props', '.prop', '> article', '.postHeader', '.web-player').each do |node|
+ node.before(node.children).remove
+ end
+
+ css('a pre', 'h3 .propType').each do |node|
+ node.name = 'code'
+ end
+
+ css('a[target]').each do |node|
+ node.remove_attribute('target')
+ end
+
+ css('center > .button', 'p:contains("short survey")', 'iframe', '.embedded-simulator', '.deprecatedIcon').remove
+
+ css('h4.methodTitle').each do |node|
+ node.name = 'h3'
+ end
+
+ css('div:not([class])', 'span:not([class])').each do |node|
+ node.before(node.children).remove
+ end
+
+ css('ul').each do |node|
+ node.before(node.children).remove if node.at_css('> p', '> h2')
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/prettier/entries.rb b/lib/docs/filters/prettier/entries.rb
new file mode 100644
index 00000000..c3411a05
--- /dev/null
+++ b/lib/docs/filters/prettier/entries.rb
@@ -0,0 +1,27 @@
+module Docs
+ class Prettier
+ class EntriesFilter < Docs::EntriesFilter
+ def get_name
+ at_css('h1').children.select(&:text?).map(&:content).join.strip
+ end
+
+ def type
+ link = at_css('.navListItemActive')
+ section = link.ancestors('.navGroup').first
+ type = section.at_css('h3').content.strip
+ return name if type == 'Configuring Prettier'
+ return name if type == 'Usage'
+ type
+ end
+
+ def additional_entries
+ entries = []
+ css('.mainContainer h2').each do |node|
+ id = node.at_css('.anchor')['id']
+ entries << [node.text, id]
+ end
+ entries
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/python/entries_v3.rb b/lib/docs/filters/python/entries_v3.rb
index 43c88e39..a28eef24 100644
--- a/lib/docs/filters/python/entries_v3.rb
+++ b/lib/docs/filters/python/entries_v3.rb
@@ -26,6 +26,13 @@ module Docs
end
def get_type
+ return 'Language Reference' if slug.start_with? 'reference'
+ return 'Python/C API' if slug.start_with? 'c-api'
+ return 'Tutorial' if slug.start_with? 'tutorial'
+
+ return 'Basics' unless slug.start_with? 'library/'
+ return 'Basics' if slug.start_with? 'library/index'
+
return 'Logging' if slug.start_with? 'library/logging'
return 'Asynchronous I/O' if slug.start_with? 'library/asyncio'
@@ -53,7 +60,7 @@ module Docs
end
def additional_entries
- return [] if root_page? || !include_default_entry? || name == 'errno'
+ return [] if root_page? || slug.start_with?('library/index') || !include_default_entry? || name == 'errno'
clean_id_attributes
entries = []
diff --git a/lib/docs/filters/r/clean_html.rb b/lib/docs/filters/r/clean_html.rb
new file mode 100644
index 00000000..8d7b0e20
--- /dev/null
+++ b/lib/docs/filters/r/clean_html.rb
@@ -0,0 +1,65 @@
+module Docs
+ class R
+ class CleanHtmlFilter < Filter
+ def call
+ slug_parts = slug.split('/')
+
+ if root_page?
+ css('a[href$="/00index"]').each do |pkg|
+ pkg['href'] = "/r-#{pkg['href'].split('/')[1]}/"
+ end
+
+ elsif slug_parts[0] == 'library'
+ title = at_css('h2')
+ title.inner_html = "#{slug_parts[3]} #{title.content}"
+
+ summary = at_css('table[summary]')
+ summary.remove if summary
+
+ css('hr ~ *, hr').remove
+
+ elsif slug_parts[-2] == 'manual'
+ css('table.menu, div.header, hr, h2.contents-heading, div.contents, table.index-cp, table.index-vr, table[summary]').remove
+
+ css('h2').each do |node|
+ node.remove if node.content.end_with? ' index'
+ end
+
+ css('span[id] + h1, span[id] + h2, span[id] + h3, span[id] + h4, span[id] + h5, span[id] + h6').each do |node|
+ # We need the first of the series of span with ids
+ span = node.previous_element
+ while span.previous
+ prev = span.previous_element
+ break unless prev.name == 'span' and prev['id']
+ span.remove
+ span = prev
+ end
+
+ node['id'] = span['id']
+ span.remove
+
+ css('div.example').each do |node|
+ node.replace(node.children)
+ end
+ end
+
+ css('h1 + h1').remove
+
+ css('.footnote h5').each do |node|
+ anchor = node.at_css('a[id]')
+ footnote = node.next_sibling
+ footnote.inner_html = "#{anchor.text} #{footnote.inner_html}"
+ footnote['id'] = anchor['id']
+ node.remove
+ end
+ end
+
+ css('pre').each do |node|
+ node['data-language'] = 'r'
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/r/entries.rb b/lib/docs/filters/r/entries.rb
new file mode 100644
index 00000000..ed09345d
--- /dev/null
+++ b/lib/docs/filters/r/entries.rb
@@ -0,0 +1,84 @@
+module Docs
+ class R
+ class EntriesFilter < Docs::EntriesFilter
+
+ PKG_INDEX_ENTRIES = Hash.new []
+
+ def call
+ if slug_parts[-1] == '00Index'
+ dir = File.dirname(result[:subpath])
+ css('tr a').each do |link|
+ PKG_INDEX_ENTRIES[link['href']] += [link.text]
+ next if link['href'] == link.text
+ context[:replace_paths][File.join(dir, "#{link.text}.html")] = File.join(dir, "#{link['href']}.html")
+ end
+ end
+
+ super
+ end
+
+ def slug_parts
+ slug.split('/')
+ end
+
+ def is_package?
+ slug_parts[0] == 'library'
+ end
+
+ def is_manual?
+ slug_parts[1] == 'manual'
+ end
+
+ def get_name
+ return at_css('h2').content if is_package?
+ title = at_css('h1.settitle')
+ title ? title.content : at_css('h1, h2').content
+ end
+
+ def get_type
+ return slug_parts[1] if is_package?
+ return at_css('h1.settitle').content if is_manual?
+ end
+
+ def include_default_entry?
+ is_package? and not slug_parts[-1] == '00Index'
+ end
+
+ def manual_section(node)
+ title = node.content.sub /^((Appendix )?[A-Z]|[0-9]+)(\.[0-9]+)* /, ''
+ title unless ['References', 'Preface', 'Acknowledgements'].include?(title) or title.end_with?(' index')
+ end
+
+ def additional_entries
+ if is_package? and slug_parts[-1] != '00Index'
+ page = slug_parts[-1]
+ return [page] + PKG_INDEX_ENTRIES.fetch(page, [])
+ end
+
+ return [] unless is_manual?
+
+ entries = []
+ unless slug_parts[-1].downcase == 'r-intro'
+ # Single top-level category
+ css('div.contents > ul a').each do |link|
+ link_name = manual_section(link)
+ entries << [link_name, link['href'].split('#')[1], name] unless link_name.nil?
+ end
+ else
+ # Split 1st level of manual into different categories
+ css('div.contents > ul > li').each do |node|
+ type = manual_section(node.at_css('a'))
+ next if type.nil?
+ node.css('> ul a').each do |link|
+ link_name = link.content.sub /^[0-9A-Z]+(\.[0-9]+)* /, ''
+ entries << [link_name, link['href'].split('#')[1], type]
+ end
+ end
+ end
+ return entries
+ end
+
+ private
+ end
+ end
+end
diff --git a/lib/docs/filters/rails/clean_html_guides.rb b/lib/docs/filters/rails/clean_html_guides.rb
index a40dcba0..0c39d137 100644
--- a/lib/docs/filters/rails/clean_html_guides.rb
+++ b/lib/docs/filters/rails/clean_html_guides.rb
@@ -24,7 +24,9 @@ module Docs
css('pre').each do |node|
code = node.at_css('code')
- language = code['class'][/highlight ?(\w+)/, 1]
+ language = code['class']
+ break if language.nil?
+ language = language [/highlight ?(\w+)/, 1]
node['data-language'] = language unless language == 'plain'
code.remove_attribute('class')
node.content = node.content.strip
diff --git a/lib/docs/filters/react_native/clean_html.rb b/lib/docs/filters/react_native/clean_html.rb
index ecc233be..5450bfa7 100644
--- a/lib/docs/filters/react_native/clean_html.rb
+++ b/lib/docs/filters/react_native/clean_html.rb
@@ -2,13 +2,30 @@ module Docs
class ReactNative
class CleanHtmlFilter < Filter
def call
- @doc = at_css('.post')
+ @doc = at_css('main .col .markdown')
if root_page?
at_css('h1').content = 'React Native Documentation'
css('h1 ~ *').remove
end
+ css('header').each do |node|
+ node.before(node.children).remove
+ end
+
+ css('.content-banner-img').remove
+ css('.anchor').remove_attribute('class')
+ css('button[aria-label="Copy code to clipboard"]').remove
+ css('h2#example').remove
+
+ css('pre').each do |node|
+ node.content = node.css('.token-line').map(&:content).join("\n")
+ node.remove_attribute('class')
+ node['data-language'] = 'jsx'
+ end
+
+ #
+
css('.docs-prevnext', '.hash-link', '.edit-page-link', '.edit-github', 'a.hash', '.edit-page-block', 'a.show', 'a.hide', 'hr').remove
css('table h1', 'table h2', 'table h3').each do |node|
@@ -21,23 +38,10 @@ module Docs
node.parent['id'] ||= node['name'] || node['id']
end
- css('.highlight').each do |node|
- node.name = 'pre'
- node.css('.gutter').remove
- node['data-language'] = node.at_css('[data-lang]').try(:[], 'data-lang') || 'js'
- node.content = node.content.strip
- end
-
css('table.highlighttable').each do |node|
node.replace(node.at_css('pre.highlight'))
end
- css('.prism').each do |node|
- node.name = 'pre'
- node['data-language'] = node['class'][/(?<=language\-)(\w+)/]
- node.content = node.content
- end
-
css('pre > code.hljs').each do |node|
node.parent['data-language'] = 'jsx'
node.before(node.children).remove
diff --git a/lib/docs/filters/react_native/entries.rb b/lib/docs/filters/react_native/entries.rb
index 71a55ece..a3a9c793 100644
--- a/lib/docs/filters/react_native/entries.rb
+++ b/lib/docs/filters/react_native/entries.rb
@@ -1,33 +1,25 @@
module Docs
class ReactNative
class EntriesFilter < Docs::EntriesFilter
- REPLACE_TYPES = {
- 'The Basics' => 'Getting Started',
- 'apis' => 'APIs',
- 'components' => 'Components'
- }
-
def get_name
at_css('h1').children.select(&:text?).map(&:content).join.strip
end
def get_type
- link = at_css('.navListItemActive')
- return 'Miscellaneous' unless link
- section = link.ancestors('.navGroup').first
- type = section.at_css('h3').content.strip
- type = REPLACE_TYPES[type] || type
- type += ": #{name}" if type == 'Components'
- type
+ type = at_css('.navbar__link--active')
+ return 'Miscellaneous' unless type
+ type.content.strip
end
def additional_entries
- css('.mainContainer h3').each_with_object [] do |node, entries|
- subname = node.text
+ css('main .col .markdown h3').each_with_object [] do |node, entries|
+ code = node.at_css('code')
+ next unless code
+ subname = code.text
next if subname.blank? || node.css('code').empty?
sep = subname.include?('()') ? '.' : '#'
subname.prepend(name + sep)
- id = node.at_css('.anchor')['id']
+ id = node['id']
entries << [subname, id]
end
end
diff --git a/lib/docs/filters/react_router/clean_html.rb b/lib/docs/filters/react_router/clean_html.rb
new file mode 100644
index 00000000..9d0828ff
--- /dev/null
+++ b/lib/docs/filters/react_router/clean_html.rb
@@ -0,0 +1,14 @@
+module Docs
+ class ReactRouter
+ class CleanHtmlFilter < Filter
+ def call
+ @doc = at_css('.md-prose')
+ css('pre').each do |node|
+ node.content = node.css('.codeblock-line').map(&:content).join("")
+ node['data-language'] = 'javascript'
+ end
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/react_router/entries.rb b/lib/docs/filters/react_router/entries.rb
new file mode 100644
index 00000000..862cf572
--- /dev/null
+++ b/lib/docs/filters/react_router/entries.rb
@@ -0,0 +1,13 @@
+module Docs
+ class ReactRouter
+ class EntriesFilter < Docs::EntriesFilter
+ def additional_entries
+ entries = []
+ css('h2[id], h3[id]').each do |node|
+ entries << [node.content, node['id'], 'API Reference']
+ end
+ entries
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/redux/clean_html.rb b/lib/docs/filters/redux/clean_html.rb
index a0aa9935..df12a319 100644
--- a/lib/docs/filters/redux/clean_html.rb
+++ b/lib/docs/filters/redux/clean_html.rb
@@ -3,26 +3,25 @@ module Docs
class CleanHtmlFilter < Filter
def call
- css('h1, h2, h3, h4').each do |node|
+ css('h1, h2, h3, h4, h5').each do |node|
node.css('a').remove
+ node.remove_attribute('class')
+ node.parent.before(node.parent.children).remove if node.parent.name == 'header'
end
css('h3').each do |node|
node['id'] = node.content.gsub(/\(|\)/, '').downcase
end
- css('.codeBlockLines_b7E3').each do |node|
- node.remove_attribute('style')
- node.name = 'pre'
+ css('pre').each do |node|
+ node.content = node.css('.token-line').map(&:content).join("\n")
node['data-language'] = 'javascript'
-
- node.css('div, span').each do |subnode|
- subnode.remove_attribute('style')
- end
-
end
- css('.copyButton_10dd').remove
+ css('*').each do |node|
+ node.remove_attribute('style')
+ node.remove if node['class'] && node['class'].include?('copyButton')
+ end
doc
diff --git a/lib/docs/filters/rust/clean_html.rb b/lib/docs/filters/rust/clean_html.rb
index 582ea8c8..fa24468d 100644
--- a/lib/docs/filters/rust/clean_html.rb
+++ b/lib/docs/filters/rust/clean_html.rb
@@ -13,7 +13,7 @@ module Docs
node.before(node.children).remove
end
else
- @doc = at_css('#main')
+ @doc = at_css('#main, #main-content')
css('.toggle-wrapper').remove
@@ -21,6 +21,11 @@ module Docs
node.content = node.at_css('.in-band').content
end
+ css('.main-heading > h1.fqn').each do |node|
+ node.parent.name = 'h1'
+ node.parent.content = node.content
+ end
+
css('.stability .stab').each do |node|
node.name = 'span'
node.content = node.content
@@ -45,6 +50,11 @@ module Docs
node.remove if node.content.include?('#[must_use]')
end
+ css('details').each do |node|
+ node.css('summary:contains("Expand description")').remove
+ node.before(node.children).remove
+ end
+
css('a.header').each do |node|
unless node.first_element_child.nil?
node.first_element_child['id'] = node['name'] || node['id']
@@ -101,8 +111,8 @@ module Docs
node.previous_element.before(node)
end
+ css('#copy-path').remove
css('.sidebar').remove
-
css('.collapse-toggle').remove
doc
diff --git a/lib/docs/filters/rust/entries.rb b/lib/docs/filters/rust/entries.rb
index 2fa5186d..176f7e02 100644
--- a/lib/docs/filters/rust/entries.rb
+++ b/lib/docs/filters/rust/entries.rb
@@ -9,7 +9,7 @@ module Docs
elsif slug == 'error-index'
'Compiler Errors'
else
- name = at_css('h1.fqn .in-band').content.remove(/\A.+\s/)
+ name = at_css('h1.fqn .in-band').content.remove(/\A.+\s/).remove('⎘')
mod = slug.split('/').first
name.prepend("#{mod}::") unless name.start_with?(mod)
name
diff --git a/lib/docs/filters/rxjs/entries.rb b/lib/docs/filters/rxjs/entries.rb
index c6e488fb..afc4ce4a 100644
--- a/lib/docs/filters/rxjs/entries.rb
+++ b/lib/docs/filters/rxjs/entries.rb
@@ -20,8 +20,9 @@ module Docs
end
def additional_entries
- css('h3[id]').map do |node|
- ["#{name}.#{node['name']}()", node['id']]
+ css('h3[id]').flat_map do |node|
+ return [] unless node['name']
+ [["#{name}.#{node['name']}()", node['id']]]
end
end
end
diff --git a/lib/docs/filters/scala/clean_html.rb b/lib/docs/filters/scala/clean_html_v2.rb
similarity index 98%
rename from lib/docs/filters/scala/clean_html.rb
rename to lib/docs/filters/scala/clean_html_v2.rb
index 0320932d..05a3ad58 100644
--- a/lib/docs/filters/scala/clean_html.rb
+++ b/lib/docs/filters/scala/clean_html_v2.rb
@@ -1,6 +1,6 @@
module Docs
class Scala
- class CleanHtmlFilter < Filter
+ class CleanHtmlV2Filter < Filter
def call
@doc = at_css('#content')
diff --git a/lib/docs/filters/scala/clean_html_v3.rb b/lib/docs/filters/scala/clean_html_v3.rb
new file mode 100644
index 00000000..28c090fd
--- /dev/null
+++ b/lib/docs/filters/scala/clean_html_v3.rb
@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+module Docs
+ class Scala
+ class CleanHtmlV3Filter < Filter
+ def call
+ # Remove unneeded elements
+ css('.documentableFilter, .documentableAnchor, .documentableBrief').remove
+
+ format_title
+ format_signature
+ format_top_links
+ format_metadata
+
+ # Remove the redundant long descriptions on the main page
+ if slug == 'index'
+ css('.contents').remove
+ else
+ format_members
+ end
+
+ simplify_html
+
+ doc
+ end
+
+ private
+
+ # Formats the title of the page
+ def format_title
+ cover_header = at_css('.cover-header')
+ return if cover_header.nil?
+
+ # Add the kind of page to the title
+ icon = cover_header.at_css('.micon')
+ types = {
+ cl: 'Class',
+ ob: 'Object',
+ tr: 'Trait',
+ en: 'Enum',
+ ty: 'Type',
+ pa: 'Package',
+ }
+ type_id = cover_header.at_css('.micon')['class']
+ type_id.remove!('micon ')
+ type_id.remove!('-wc')
+ type = types[type_id.to_sym]
+ name = CGI.escapeHTML cover_header.at_css('h1').text
+
+ # Add the package name
+ package = at_css('.breadcrumbs a:nth-of-type(3)').text
+ package = package + '.' unless name.empty? || package.empty?
+
+ # Replace the title
+ title = root_page? ? 'Package root' : "#{type} #{package}#{name}".strip
+ cover_header.replace "#{title}
"
+ end
+
+ # Formats the signature block at the top of the page
+ def format_signature
+ signature = at_css('.signature')
+ signature_annotations = signature.at_css('.annotations')
+ signature_annotations.name = 'small' unless signature_annotations.nil?
+ signature.replace "#{signature.inner_html}
"
+ end
+
+ # Formats the top links (companion page, source code)
+ def format_top_links
+ # Companion page (e.g. List ↔ List$)
+ links = []
+ at_css('.attributes').css('dt').each do |dt|
+ next if dt.content.strip != 'Companion:'
+ dd = dt.next_sibling
+
+ companion_link = dd.at_css('a')
+ companion_link.content = "Companion #{companion_link.content}"
+ links.append(companion_link.to_html)
+
+ dt.remove
+ dd.remove
+ end
+
+ # Source code
+ at_css('.attributes').css('dt').each do |dt|
+ next if dt.content.strip != 'Source:'
+ dd = dt.next_sibling
+
+ source_link = dd.at_css('a')
+ source_link.content = 'Source code'
+ links.append(source_link.to_html)
+
+ dt.remove
+ dd.remove
+ end
+
+ # Format the links
+ title = at_css('h1')
+ title.add_next_sibling("#{links.join(' • ')}")
+ end
+
+ # Metadata about the whole file (e.g. supertypes)
+ def format_metadata
+ # Format the values
+ css('.tabs.single .monospace').each do |node|
+ node.css('> div').each do |div|
+ div['class'] = 'member'
+ end
+
+ node['class'] = 'related-types'
+
+ if node.children.count > 15 # Hide too large lists
+ node.replace "
+ #{node.children.count} types
+ #{node.to_html}
+ "
+ end
+ end
+
+ attributes = at_css('.attributes')
+
+ # Change the HTML structure
+ tabs_names = css('.tabs.single .names .tab')
+ tabs_contents = css('.tabs.single .contents .tab')
+ tabs_names.zip(tabs_contents).each do |name, contents|
+ next if name.content == "Graph"
+
+ attributes.add_child("- #{name.content}
")
+ attributes.add_child("- #{contents.inner_html.strip}
")
+ end
+
+ convert_dl_to_table(attributes)
+
+ tabs = at_css('.tabs')
+ tabs.remove unless tabs.nil? || tabs.parent['class'] == 'membersList'
+ end
+
+ # Format the members (methods, values…)
+ def format_members
+ # Section headings
+ css('.cover h2').each do |node|
+ node.name = 'h3'
+ end
+ css('h2:not(#signature)').remove
+ css(
+ '.membersList h3',
+
+ # Custom group headers for which Scaladoc generates invalid HTML
+ # (…
)
+ '.documentableList > h3:empty + p'
+ ).each do |node|
+ node.name = 'h2'
+ node.content = node.content
+ end
+
+ # Individual members
+ css('.documentableElement').each do |element|
+ header = element.at_css('.header')
+ header.name = 'h3'
+
+ id = element['id']
+ element.remove_attribute('id')
+ header['id'] = id unless id.nil?
+
+ annotations = element.at_css('.annotations')
+ annotations.name = 'small'
+ header.prepend_child(annotations)
+
+ # View source
+ element.css('dt').each do |dt|
+ next if dt.content.strip != 'Source:'
+ dd = dt.next_sibling
+
+ source_link = dd.at_css('a')
+ source_link.content = 'Source'
+ source_link['class'] = 'source-link'
+ header.prepend_child(source_link)
+
+ dt.remove
+ dd.remove
+ end
+
+ # Format attributes as a table
+ dl = element.at_css('.attributes')
+ convert_dl_to_table(dl) unless dl.nil?
+
+ # Remove the unnecessary wrapper element
+ element.replace(element.inner_html)
+ end
+
+ # Remove deprecated sections
+ css('.documentableList').each do |list|
+ header = list.at_css('.groupHeader')
+ list.remove if (header.text.downcase.include? 'deprecate' rescue false)
+ end
+
+ # Code blocks
+ css('pre > code').each do |code|
+ pre = code.parent
+ pre['data-language'] = 'scala'
+ pre.inner_html = code.inner_html
+ end
+ end
+
+ # Simplify the HTML structure by removing useless elements
+ def simplify_html
+ # Remove unneeded parts of the document
+ @doc = at_css('#content > div')
+
+ # Remove the useless elements around members
+ css('.documentableList > *').each do |element|
+ element.parent = doc
+ end
+ at_css('.membersList').remove
+
+ # Remove useless classes
+ css('.header, .groupHeader, .cover, .documentableName').each do |element|
+ element.remove_attribute('class')
+ end
+
+ # Remove useless attributes
+ css('[t]').each do |element|
+ element.remove_attribute('t')
+ end
+
+ # Remove useless wrapper elements
+ css('.docs, .doc, .memberDocumentation, span, div:not([class])').each do |element|
+ element.replace(element.children)
+ end
+ end
+
+ def convert_dl_to_table(dl)
+ table = Nokogiri::XML::Node.new('table', doc)
+ table['class'] = 'attributes'
+
+ dl.css('> dt').each do |dt|
+ dd = dt.next_element
+ has_dd = dd.name == 'dd' rescue false
+
+ tr = Nokogiri::XML::Node.new('tr', doc)
+ colspan = has_dd ? '' : ' colspan="2"' # handle - without following
-
+ tr.add_child("
#{dt.inner_html.sub(/:$/, '')}")
+
+ tr.add_child("#{dd.inner_html} ") if has_dd
+
+ table.add_child(tr)
+ end
+
+ dl.replace(table)
+ end
+
+ end
+ end
+end
diff --git a/lib/docs/filters/scala/entries.rb b/lib/docs/filters/scala/entries_v2.rb
similarity index 95%
rename from lib/docs/filters/scala/entries.rb
rename to lib/docs/filters/scala/entries_v2.rb
index 5eff47fb..ca3b6d12 100644
--- a/lib/docs/filters/scala/entries.rb
+++ b/lib/docs/filters/scala/entries_v2.rb
@@ -1,6 +1,6 @@
module Docs
class Scala
- class EntriesFilter < Docs::EntriesFilter
+ class EntriesV2Filter < Docs::EntriesFilter
REPLACEMENTS = {
'$eq' => '=',
'$colon' => ':',
@@ -75,12 +75,12 @@ module Docs
# include the companion object.
def package_name
name = package_drop_last(slug_parts)
- name.empty? ? '_root_' : name
+ name.empty? ? 'scala' : name
end
def parent_package
parent = package_drop_last(package_name.split('.'))
- parent.empty? ? '_root_' : parent
+ parent.empty? ? 'scala' : parent
end
def package_drop_last(parts)
diff --git a/lib/docs/filters/scala/entries_v3.rb b/lib/docs/filters/scala/entries_v3.rb
new file mode 100644
index 00000000..62337446
--- /dev/null
+++ b/lib/docs/filters/scala/entries_v3.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Docs
+ class Scala
+ class EntriesV3Filter < Docs::EntriesFilter
+ REPLACEMENTS = {
+ '$eq' => '=',
+ '$colon' => ':',
+ '$less' => '<',
+ }
+
+ def get_name
+ if is_package?
+ at_css('.cover-header h1').text
+ else
+ name = slug.split('/').last
+
+ # Some objects have inner objects, show ParentObject$.ChildObject$ instead of ParentObject$$ChildObject$
+ name = name.gsub('$$', '$.')
+
+ REPLACEMENTS.each do |key, value|
+ name = name.gsub(key, value)
+ end
+
+ # If a dollar sign is used as separator between two characters, replace it with a dot
+ name.gsub(/([^$.])\$([^$.])/, '\1.\2')
+ end
+ end
+
+ def get_type
+ # if this entry is for a package, we group the package under the parent package
+ if is_package?
+ parent_package
+ # otherwise, group it under the regular package name
+ else
+ package_name
+ end
+ end
+
+ def include_default_entry?
+ # Ignore package pages
+ at_css('.cover-header .micon.pa').nil?
+ end
+
+ def additional_entries
+ entries = []
+ titles = []
+
+ css(".documentableElement").each do |node|
+ # Ignore elements without IDs
+ id = node['id']
+ next if id.nil?
+
+ # Ignore deprecated and inherited members
+ next unless node.at_css('.deprecated').nil?
+
+ member_name = node.at_css('.documentableName').content
+ title = "#{name}.#{member_name}"
+
+ # Add () to methods that take parameters, i.e. methods who have (…)
+ # in their signature, ignoring occurrences of (implicit …) and (using …)
+ signature = node.at_css('.signature').content
+ title += '()' if signature =~ /\((?!implicit)(?!using ).*\)/
+
+ next if titles.include?(title) # Ignore duplicates (function overloading)
+
+ entries << [title, id]
+ titles.push(title)
+ end
+
+ entries
+ end
+
+ private
+
+ # For the package name, we use the slug rather than parsing the package
+ # name from the HTML because companion object classes may be broken out into
+ # their own entries (by the source documentation). When that happens,
+ # we want to group these classes (like `scala.reflect.api.Annotations.Annotation`)
+ # under the package name, and not the fully-qualfied name which would
+ # include the companion object.
+ def package_name
+ name = package_drop_last(slug_parts)
+ name.empty? ? 'scala' : name
+ end
+
+ def parent_package
+ parent = package_drop_last(package_name.split('.'))
+ parent.empty? ? 'scala' : parent
+ end
+
+ def package_drop_last(parts)
+ parts[0...-1].join('.')
+ end
+
+ def slug_parts
+ slug.split('/')
+ end
+
+ def is_package?
+ !at_css('.cover-header .micon.pa').nil?
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/sphinx/clean_html.rb b/lib/docs/filters/sphinx/clean_html.rb
index 71720aca..048f6d3b 100644
--- a/lib/docs/filters/sphinx/clean_html.rb
+++ b/lib/docs/filters/sphinx/clean_html.rb
@@ -58,6 +58,8 @@ module Docs
end
css('dt').each do |node|
+ next if current_url.host == 'matplotlib.org'
+ next if current_url.host == 'numpy.org'
next unless node['id'] || node.at_css('code, .classifier')
links = []
links << node.children.last.remove while node.children.last.try(:name) == 'a'
diff --git a/lib/docs/filters/spring_boot/clean_html.rb b/lib/docs/filters/spring_boot/clean_html.rb
index eb1639d3..672fc912 100644
--- a/lib/docs/filters/spring_boot/clean_html.rb
+++ b/lib/docs/filters/spring_boot/clean_html.rb
@@ -2,7 +2,9 @@ module Docs
class SpringBoot
class CleanHtmlFilter < Filter
def call
-
+ at_css('#content').prepend_child(at_css('h1'))
+ @doc = at_css('#content')
+
css('pre').each do |node|
language = node.children.first['data-lang'] if node.children.first.name == 'code'
node['data-language'] = language
diff --git a/lib/docs/filters/sqlite/clean_html.rb b/lib/docs/filters/sqlite/clean_html.rb
index 5407677d..b9cd3693 100644
--- a/lib/docs/filters/sqlite/clean_html.rb
+++ b/lib/docs/filters/sqlite/clean_html.rb
@@ -90,6 +90,8 @@ module Docs
node.remove_attribute('fill')
elsif node['style'] == 'fill:none;stroke-width:2.16;stroke:rgb(0,0,0);'
node.add_class('stroke')
+ elsif node['style'] == 'fill:none;stroke-width:2.16;stroke-linejoin:round;stroke:rgb(0,0,0);'
+ node.add_class('stroke')
elsif node['style'] == 'fill:none;stroke-width:3.24;stroke:rgb(211,211,211);'
node.add_class('stroke')
elsif node['style']
diff --git a/lib/docs/filters/tailwindcss/clean_html.rb b/lib/docs/filters/tailwindcss/clean_html.rb
new file mode 100644
index 00000000..9815cf09
--- /dev/null
+++ b/lib/docs/filters/tailwindcss/clean_html.rb
@@ -0,0 +1,114 @@
+module Docs
+ class Tailwindcss
+ class CleanHtmlFilter < Filter
+ def call
+ # Remove main page headers (top level sticky)
+ css('#__next > .sticky').remove
+ # And anything absolutely positioned (fancy floating navigation elements we don't care about)
+ css('#__next .absolute').remove
+ # Remove the left-navigation we scraped
+ css('nav').remove
+
+ css('svg').remove if root_page?
+
+ # Remove the duplicate category name at the top of the page - redundant
+ at_css('header#header > div:first-child > p:first-child').remove
+
+ # Remove the right navigation sidebar
+ at_css('header#header').parent.css('> div:has(h5:contains("On this page"))').remove
+
+ # Remove footer + prev/next navigation
+ at_css('footer').remove
+
+ # Handle long lists of class reference that otherwise span several scrolled pages
+ if class_reference = at_css('#class-reference')
+ reference_container = class_reference.parent
+ classes_container = reference_container.children.reject{ |child| child == class_reference }.first
+
+ rows = classes_container.css("tr")
+
+ if rows.length > 10
+ classes_container.set_attribute("class", "long-quick-reference")
+ end
+ end
+
+ # Remove border color preview column as it isn't displayed anyway
+ if result[:path] == "border-color" and class_reference = at_css('#class-reference')
+ class_reference.parent.css("thead th:nth-child(3)").remove
+ class_reference.parent.css("tbody td:nth-child(3)").remove
+ end
+
+ if result[:path] == "customizing-colors"
+ # It's nice to be able to quickly find out what a color looks like
+ css('div[style^="background-color:"]').each do |node|
+ node.set_attribute("class", "color-swatch")
+
+ swatch_container = node.parent
+ swatch_container.set_attribute("class", "color-swatch-container")
+ swatch_container.children.reject{ |child| child == node }.first.set_attribute("class", "color-tone-information")
+
+ swatch_group = swatch_container.parent
+ swatch_group.set_attribute("class", "color-swatch-group")
+
+ color = swatch_group.parent
+ color.set_attribute("class", "color")
+
+ color_list = color.parent.parent
+ color_list.set_attribute("class", "colors")
+ end
+ end
+
+ # Remove buttons to expand lists - those are already expanded and the button is useless
+ css('div > button:contains("Show all classes")').each do |node|
+ node.parent.remove
+ end
+
+ # Remove class examples - not part of devdocs styleguide? (similar to bootstrap)
+ # Refer to https://github.com/freeCodeCamp/devdocs/pull/1534#pullrequestreview-649818936
+ css('.not-prose').each do |node|
+ if node.parent.children.length == 1
+ node.parent.remove
+ else
+ node.remove
+ end
+ end
+
+ # Properly format code examples
+ css('pre > code:first-child').each do |node|
+ node.parent['data-language'] = node['class'][/language-(\w+)/, 1] if node['class'] and node['class'][/language-(\w+)/]
+ node.parent.content = node.parent.content
+ end
+
+ @doc.traverse { |node| cleanup_tailwind_classes(node) }
+
+ #remove weird
(https://github.com/damms005/devdocs/commit/8c9fbd859b71a2525b94a35ea994393ce2b6fedb#commitcomment-50091018)
+ css('hr').remove
+
+ doc
+ end
+
+ # Removes all classes not allowlisted in the below semantic_classes array - such as tailwinds utility classes
+ def cleanup_tailwind_classes(node)
+ class_name = node.attr("class")
+
+ if class_name == nil
+ return node.children.each { |child| cleanup_tailwind_classes(child) }
+ end
+
+ semantic_classes = ["code", "color-swatch", "color-swatch-container", "color-tone-information", "color-swatch-group", "color", "colors", "long-quick-reference"]
+
+ classes = class_name.split.select do |klas|
+ semantic_classes.include? klas
+ end
+
+ if classes.length === 0
+ node.delete("class")
+ else
+ node.set_attribute("class", classes.join(" "))
+ end
+
+ node.children.each { |child| cleanup_tailwind_classes(child) }
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/tailwindcss/entries.rb b/lib/docs/filters/tailwindcss/entries.rb
new file mode 100644
index 00000000..ad0bf7b4
--- /dev/null
+++ b/lib/docs/filters/tailwindcss/entries.rb
@@ -0,0 +1,24 @@
+module Docs
+ class Tailwindcss
+ class EntriesFilter < Docs::EntriesFilter
+ def get_type
+ # We are only interested in children list items
+ selector = "nav li li a[href='#{result[:path]}']"
+
+ anchor = at_css(selector)
+ category_list = anchor.ancestors('li')[1]
+ title = category_list.css('h5')
+
+ return title.inner_text
+ end
+
+ def get_name
+ # We are only interested in children list items
+ selector = "nav li li a[href='#{result[:path]}']"
+ item = at_css(selector)
+
+ return item.inner_text
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/tailwindcss/noop.rb b/lib/docs/filters/tailwindcss/noop.rb
new file mode 100644
index 00000000..1c89e2e3
--- /dev/null
+++ b/lib/docs/filters/tailwindcss/noop.rb
@@ -0,0 +1,9 @@
+module Docs
+ class Tailwindcss
+ class NoopFilter < Filter
+ def call
+ return html
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/tensorflow/clean_html.rb b/lib/docs/filters/tensorflow/clean_html.rb
index e508f971..6d594d70 100644
--- a/lib/docs/filters/tensorflow/clean_html.rb
+++ b/lib/docs/filters/tensorflow/clean_html.rb
@@ -5,6 +5,7 @@ module Docs
@doc = at_css('.devsite-article')
css('hr', '.devsite-nav', '.devsite-content-footer', '.devsite-article-body > br', '.devsite-article-meta', 'devsite-nav-buttons', '.devsite-banner', '.tfo-api img', '.tfo-notebook-buttons img', '.tfo-notebook-buttons>:first-child').remove
+ css('devsite-bookmark').remove
css('aside.note').each do |node|
node.name = 'blockquote'
diff --git a/lib/docs/filters/typescript/clean_html.rb b/lib/docs/filters/typescript/clean_html.rb
index 8bcc1bd2..f7ba2b33 100644
--- a/lib/docs/filters/typescript/clean_html.rb
+++ b/lib/docs/filters/typescript/clean_html.rb
@@ -30,13 +30,15 @@ module Docs
@doc = at_css('article > .whitespace > .markdown')
doc.child.before(deprecated) if deprecated
else # tsconfig page
- @doc = at_css('.markdown > div')
-
- at_css('h2').remove
end
css('.anchor', 'a:contains("Try")', 'h2 a', 'h3 a', 'svg', '#full-option-list').remove
+ # tsconfig page
+ css('.markdown', '.compiler-option', '.compiler-option-md', '.compiler-content').each do |node|
+ node.remove_attribute('class')
+ end
+
css('pre').each do |node|
language = node.at_css('.language-id') ? node.at_css('.language-id').content : 'typescript'
node.css('.language-id').remove
diff --git a/lib/docs/filters/typescript/entries.rb b/lib/docs/filters/typescript/entries.rb
index 8073f3ea..c2ad3b0e 100644
--- a/lib/docs/filters/typescript/entries.rb
+++ b/lib/docs/filters/typescript/entries.rb
@@ -14,6 +14,7 @@ module Docs
)
def get_name
+ return 'TSConfig Reference' if slug == 'tsconfig'
at_css('h1') ? at_css('h1').content : at_css('h2').content
end
@@ -27,6 +28,7 @@ module Docs
def additional_entries
return [] if DEPRECATED_PAGES.include? slug
+ return [] if slug == 'tsconfig-json'
base_url.path == '/' ? tsconfig_entries : handbook_entries
end
@@ -38,7 +40,7 @@ module Docs
def handbook_entries
css('h2').each_with_object [] do |node, entries|
- entries << [node.content, node['id']]
+ entries << ["#{name}: #{node.content}", node['id']] if node['id']
end
end
diff --git a/lib/docs/filters/vite/clean_html.rb b/lib/docs/filters/vite/clean_html.rb
new file mode 100644
index 00000000..5eae8dce
--- /dev/null
+++ b/lib/docs/filters/vite/clean_html.rb
@@ -0,0 +1,50 @@
+module Docs
+ class Vite
+ class CleanHtmlFilter < Filter
+ def call
+ return 'Vite
' if root_page?
+ @doc = at_css('main .content > div')
+
+ css('.demo', '.guide-links', '.footer', '#ad').remove
+ css('.header-anchor', '.page-edit', '.page-nav').remove
+
+ css('.custom-block-title').each do |node|
+ node.name = 'strong'
+ end
+
+ # Remove CodePen div
+ css('.codepen').each do |node|
+ next if node.previous_element.nil?
+ span = node.css('span:contains("See the Pen")').remove
+ node.previous_element.add_child(' ')
+ node.previous_element.add_child(span)
+ node.remove
+ end
+
+ # Remove code highlighting
+ css('figure').each do |node|
+ node.name = 'pre'
+ node.content = node.at_css('td.code pre').css('.line').map(&:content).join("\n")
+ node['data-language'] = node['class'][/highlight (\w+)/, 1]
+ end
+
+ css('.line-numbers-wrapper').remove
+ css('pre').each do |node|
+ node.content = node.content.strip
+ node['data-language'] = 'javascript'
+ end
+
+ css('iframe').each do |node|
+ node['sandbox'] = 'allow-forms allow-scripts allow-same-origin'
+ node.remove if node['src'][/player.vimeo.com/] # https://v3.vuejs.org/guide/migration/introduction.html#overview
+ end
+
+ css('details').each do |node|
+ node.name = 'div'
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/vite/entries.rb b/lib/docs/filters/vite/entries.rb
new file mode 100644
index 00000000..99c5762c
--- /dev/null
+++ b/lib/docs/filters/vite/entries.rb
@@ -0,0 +1,23 @@
+module Docs
+ class Vite
+ class EntriesFilter < Docs::EntriesFilter
+ def get_name
+ name = at_css('h1').content
+ name.sub! %r{\s*#\s*}, ''
+ name
+ end
+
+ def get_type
+ at_css('header nav .item.active').content.strip
+ end
+
+ def additional_entries
+ css('h2, h3').each_with_object [] do |node, entries|
+ type = node.content.strip
+ type.sub! %r{\s*#\s*}, ''
+ entries << ["#{name}: #{type}", node['id']]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/vue/clean_html.rb b/lib/docs/filters/vue/clean_html.rb
index affde245..288e7bf3 100644
--- a/lib/docs/filters/vue/clean_html.rb
+++ b/lib/docs/filters/vue/clean_html.rb
@@ -2,18 +2,34 @@ module Docs
class Vue
class CleanHtmlFilter < Filter
def call
- @doc = at_css(version == '3' ? 'main' : '.content')
+ if current_url.host == 'vitejs.dev'
+ return 'Vite
' if root_page?
+ @doc = at_css('.content > div')
+ else
+ return 'Vue.js
' if root_page?
+ @doc = at_css(version == '3' ? 'main > div > div' : '.content')
+ end
at_css('h1').content = 'Vue.js' if root_page?
doc.child.before('Vue.js API
') if slug == 'api/' || slug == 'api/index'
css('.demo', '.guide-links', '.footer', '#ad').remove
css('.header-anchor', '.page-edit', '.page-nav').remove
+ css('.next-steps').remove
css('.custom-block-title').each do |node|
node.name = 'strong'
end
+ # Remove CodePen div
+ css('.codepen').each do |node|
+ next if node.previous_element.nil?
+ span = node.css('span:contains("See the Pen")').remove
+ node.previous_element.add_child(' ')
+ node.previous_element.add_child(span)
+ node.remove
+ end
+
# Remove code highlighting
css('figure').each do |node|
node.name = 'pre'
diff --git a/lib/docs/filters/vue/entries_v3.rb b/lib/docs/filters/vue/entries_v3.rb
index a2941e3a..948e72e1 100644
--- a/lib/docs/filters/vue/entries_v3.rb
+++ b/lib/docs/filters/vue/entries_v3.rb
@@ -8,23 +8,32 @@ module Docs
'Style Guide'
else
name = at_css('h1').content
- name.sub! %r{#\s*}, ''
- index = css('.sidebar-link').to_a.index(at_css('.sidebar-link.active'))
- name.prepend "#{index + 1}. " if index
+ name = _fix_name(name)
+ subtype = at_css('.title-text.active')
+ index = css('aside nav .link').to_a.index(at_css('aside nav .link.active'))
+ name.prepend "#{index + 1}. " if index && !slug.start_with?('api') && !slug.start_with?('style-guide')
+ name.concat " (#{subtype.content.strip})" if subtype && slug.start_with?('guide')
name
end
end
+ def _fix_name(name)
+ name.sub! %r{#\s*}, ''
+ name.sub! %r{\s*3\.\d\+$}, ''
+ name
+ end
+
def get_type
if slug.start_with?('guide/migration')
'Migration'
- elsif slug.start_with?('guide')
- subtype = at_css('.sidebar-heading.open, .sidebar-link.active')
- subtype ? "Guide: #{subtype.content}": 'Guide'
- elsif slug == 'style-guide/'
+ elsif slug.start_with?('guide') or root_page?
+ 'Guide'
+ elsif slug.start_with?('style-guide')
'Style Guide'
else
- 'API'
+ title = at_css('.title-text.active').content.strip
+ title = _fix_name(title)
+ "API: #{title}"
end
end
@@ -32,21 +41,21 @@ module Docs
return [] if slug.start_with?('guide')
type = nil
- css('h2, h3').each_with_object [] do |node, entries|
+ at_css('main').css('h2, h3').each_with_object [] do |node, entries|
if node.name == 'h2'
type = node.content.strip
- type.sub! %r{#\s*}, ''
+ type = _fix_name(type)
next if slug == 'style-guide/'
- title = at_css('h1').content.strip
- title.sub! %r{#\s*}, ''
- entries << [type, node['id'], "API: #{title}"]
+ name = "#{get_type}: #{type}"
+ name.sub! %r{^API: }, ''
+ entries << [name, node['id'], get_type]
elsif slug == 'style-guide/'
+ next if node['id'].match(/rule-categories|priority-/)
name = node.content.strip
- name.sub! %r{#\s*}, ''
+ name = _fix_name(name)
name.sub! %r{\(.*\)}, '()'
name.sub! /(essential|strongly recommended|recommended|use with caution)\Z/, ''
- curent_type = "Style Guide: #{type.sub(/Rules: /, ': ')}"
- entries << [name, node['id'], curent_type]
+ entries << [name, node['id'], 'Style Guide']
end
end
end
diff --git a/lib/docs/filters/vue_router/entries.rb b/lib/docs/filters/vue_router/entries.rb
index b38fed28..11c5e8a0 100644
--- a/lib/docs/filters/vue_router/entries.rb
+++ b/lib/docs/filters/vue_router/entries.rb
@@ -3,7 +3,7 @@ module Docs
class EntriesFilter < Docs::EntriesFilter
def get_name
name = at_css('h1').content
- name.remove! '# '
+ name.sub! %r{#\s*}, ''
name
end
@@ -34,9 +34,9 @@ module Docs
title = node
title = title.previous_element until title.name == 'h2'
title = title.content.strip
- title.remove! '# '
+ title.sub! %r{#\s*}, ''
- entry_name.remove! '# '
+ entry_name.sub! %r{#\s*}, ''
case title
when 'Router Construction Options'
diff --git a/lib/docs/filters/vuex/entries.rb b/lib/docs/filters/vuex/entries.rb
index 04846fd2..cb037399 100644
--- a/lib/docs/filters/vuex/entries.rb
+++ b/lib/docs/filters/vuex/entries.rb
@@ -4,7 +4,7 @@ module Docs
def get_name
name = at_css('h1').content
- name.remove! '# '
+ name.sub! %r{#\s*}, ''
# Add index on guides
unless subpath.start_with?('api')
@@ -42,9 +42,9 @@ module Docs
title = node
title = title.previous_element until title.name == 'h2'
title = title.content.strip
- title.remove! '# '
+ title.sub! %r{#\s*}, ''
- entry_name.remove! '# '
+ entry_name.sub! %r{#\s*}, ''
unless entry_name.start_with?('router.')
case title
diff --git a/lib/docs/filters/webpack/clean_html.rb b/lib/docs/filters/webpack/clean_html.rb
index ebc235f5..b57190a9 100644
--- a/lib/docs/filters/webpack/clean_html.rb
+++ b/lib/docs/filters/webpack/clean_html.rb
@@ -20,6 +20,10 @@ module Docs
node.parent.content = node.parent.content
end
+ css('*').each do |node|
+ node.remove if node['class'] && node['class'][/print:hidden/]
+ end
+
# for webpack-contrib /loaders and /plugins
shields = at_css('img[src*="shields.io"]')
shields.ancestors('p')[0].remove if shields
diff --git a/lib/docs/filters/webpack/entries.rb b/lib/docs/filters/webpack/entries.rb
index cd943a72..a293286f 100644
--- a/lib/docs/filters/webpack/entries.rb
+++ b/lib/docs/filters/webpack/entries.rb
@@ -22,16 +22,16 @@ module Docs
def additional_entries
if slug.start_with?('configuration')
- css('h2[id] code').each_with_object [] do |node, entries|
- next if node.previous.try(:content).present?
- entries << [node.content, node.parent['id']]
+ css('h2 > [id]').each_with_object [] do |node, entries|
+ next if version.to_f < 5 && node.previous.try(:content).present?
+ entries << [node.parent.content, node['id']]
end
elsif slug.start_with?('api') && slug != 'api/parser'
css('.header[id] code').each_with_object [] do |node, entries|
- next if node.previous.try(:content).present? || node.next.try(:content).present?
+ next if version.to_f < 5 && (node.previous.try(:content).present? || node.next.try(:content).present?)
name = node.content.sub(/\(.*\)/, '()')
name.prepend "#{self.name.split(':').first}: "
- entries << [name, node.parent['id']]
+ entries << [name, node['id']]
end
else
[]
diff --git a/lib/docs/filters/wordpress/clean_html.rb b/lib/docs/filters/wordpress/clean_html.rb
index 32cf3b3f..a7aeb472 100644
--- a/lib/docs/filters/wordpress/clean_html.rb
+++ b/lib/docs/filters/wordpress/clean_html.rb
@@ -14,24 +14,32 @@ module Docs
'.anchor', '.toc-jump', '.source-code-links', '.user-notes',
'.show-more', '.hide-more').remove
- br = //i
-
header = at_css('h1')
header.content = header.content.strip
doc.prepend_child header
+ # Remove permalink
+ css('h2 > a, h3 > a').each do |node|
+ node.parent.remove_attribute('class')
+ node.parent.remove_attribute('tabindex')
+ node.parent.content = node.content
+ end
+
# Add PHP code highlighting
css('pre').each do |node|
node['data-language'] = 'php'
end
css('.source-code-container').each do |node|
+ node.remove_class('source-code-container')
node.name = 'pre'
- node.inner_html = node.inner_html.gsub(br, "\n")
+ node.inner_html = node.inner_html.gsub(//i, "\n")
node.content = node.content.strip
node['data-language'] = 'php'
end
+ css('section').remove_attribute('class')
+
doc
end
end
diff --git a/lib/docs/filters/yarn/clean_html_berry.rb b/lib/docs/filters/yarn/clean_html_berry.rb
new file mode 100644
index 00000000..96b3ee53
--- /dev/null
+++ b/lib/docs/filters/yarn/clean_html_berry.rb
@@ -0,0 +1,48 @@
+module Docs
+ class Yarn
+ class CleanHtmlBerryFilter < Filter
+ def call
+ if slug.empty?
+ @doc = at_css('main')
+ css(
+ (['div:first-child'] * 3).join('>'), # Tagline
+ 'img',
+ 'hr', # Footer
+ 'hr + div', # Footer
+ ).remove
+
+ css('a').each do |link|
+ link.name = 'div'
+ link.css('h3').each do |node|
+ node.replace("#{node.content}
")
+ end
+ end
+
+ return doc
+ end
+
+ @doc = at_css('article')
+ # Heading & edit link
+ css('h1', 'h1 + a').remove unless slug.start_with?('configuration')
+
+ if slug.start_with?('cli')
+ css('.header-code').each do |node|
+ node.name = 'span'
+ end
+ end
+
+ if slug.start_with?('configuration')
+ css('h1', 'h2').each do |node|
+ node.name = node.name.sub(/\d/) { |i| i.to_i + 1 }
+ end
+ end
+
+ css('*').each do |node|
+ node.remove_attribute('style')
+ end
+
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/yarn/entries.rb b/lib/docs/filters/yarn/entries.rb
index 809bb1a6..ebd65838 100644
--- a/lib/docs/filters/yarn/entries.rb
+++ b/lib/docs/filters/yarn/entries.rb
@@ -13,8 +13,7 @@ module Docs
def get_type
type = at_css('.guide-nav a').content.strip
- type.remove! ' Introduction'
- type
+ type.sub 'CLI Introduction', 'CLI Commands'
end
end
end
diff --git a/lib/docs/filters/yarn/entries_berry.rb b/lib/docs/filters/yarn/entries_berry.rb
new file mode 100644
index 00000000..44c1e18e
--- /dev/null
+++ b/lib/docs/filters/yarn/entries_berry.rb
@@ -0,0 +1,28 @@
+module Docs
+ class Yarn
+ class EntriesBerryFilter < Docs::EntriesFilter
+ def get_name
+ if slug.start_with?('configuration')
+ filename = at_css('main .active code')
+ content = filename.content
+ return filename.parent.content.sub content, " (#{content})"
+ end
+
+ name = at_css('h1').content
+
+ if slug.start_with?('getting-started')
+ active_link = at_css('main .active')
+ links = active_link.parent.children.to_a
+ name.prepend "#{links.index(active_link) + 1}. "
+ end
+
+ name
+ end
+
+ def get_type
+ return 'CLI' if slug.start_with?('sdks', 'pnpify')
+ at_css('header .active').content
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/zig/clean_html.rb b/lib/docs/filters/zig/clean_html.rb
new file mode 100644
index 00000000..df2601f8
--- /dev/null
+++ b/lib/docs/filters/zig/clean_html.rb
@@ -0,0 +1,19 @@
+module Docs
+ class Zig
+ class CleanHtmlFilter < Filter
+ def call
+ at_css('main').prepend_child at_css('h1')
+ @doc = at_css('main')
+ css('a.hdr').remove
+ css('h1, h2, h3').each do |node|
+ node.content = node.content
+ end
+ css('pre > code').each do |node|
+ node.parent['data-language'] = 'zig'
+ node.parent.content = node.parent.content
+ end
+ doc
+ end
+ end
+ end
+end
diff --git a/lib/docs/filters/zig/entries.rb b/lib/docs/filters/zig/entries.rb
new file mode 100644
index 00000000..73979b01
--- /dev/null
+++ b/lib/docs/filters/zig/entries.rb
@@ -0,0 +1,25 @@
+module Docs
+ class Zig
+ class EntriesFilter < Docs::EntriesFilter
+ def additional_entries
+ entries = []
+ type = nil
+ subtype = nil
+
+ css('h2, h3').each do |node|
+ if node.name == 'h2' && node['id']
+ type = node.content.gsub(/ §/, '')
+ subtype = nil
+ entries << [type, node['id'], type]
+ elsif node.name == 'h3' && node['id']
+ subtype = node.content.gsub(/ §/, '')
+ name = "#{type}: #{subtype}"
+ entries << [name, node['id'], type]
+ end
+ end
+
+ entries
+ end
+ end
+ end
+end
diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb
index 3654bbad..ef653c53 100644
--- a/lib/docs/scrapers/angular.rb
+++ b/lib/docs/scrapers/angular.rb
@@ -7,14 +7,28 @@ module Docs
home: 'https://angular.io/',
code: 'https://github.com/angular/angular'
}
+ self.base_url = 'https://angular.io/'
+ self.root_path = 'docs'
+
+ html_filters.push 'angular/clean_html', 'angular/entries'
options[:max_image_size] = 256_000
options[:attribution] = <<-HTML
- © 2010–2021 Google, Inc.
+ © 2010–2022 Google, Inc.
Licensed under the Creative Commons Attribution License 4.0.
HTML
+ options[:follow_links] = false
+ options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
+ options[:fix_urls_before_parse] = ->(url) do
+ url.sub! %r{\Aguide/}, '/guide/'
+ url.sub! %r{\Atutorial/}, '/tutorial/'
+ url.sub! %r{\Aapi/}, '/api/'
+ url.sub! %r{\Agenerated/}, '/generated/'
+ url
+ end
+
module Common
private
@@ -46,195 +60,91 @@ module Docs
def handle_response(response)
if response.mime_type.include?('json')
begin
- response.options[:response_body] = JSON.parse(response.body)['contents']
+ json = JSON.parse(response.body)
+ response.options[:response_body] = json['contents']
+ response.url.path = response.url.path.gsub(/generated\/docs\/.*/, json['id'])
+ response.effective_url.path = response.effective_url.path.gsub(/generated\/docs\/.*/, json['id'])
rescue JSON::ParserError
response.options[:response_body] = ''
end
response.headers['Content-Type'] = 'text/html'
- response.url.path = response.url.path.sub('/generated/docs/', '/').remove('.json')
- response.effective_url.path = response.effective_url.path.sub('/generated/docs/', '/').remove('.json')
end
super
end
end
+ module Since12
+ def url_for(path)
+ # See encodeToLowercase im aio/tools/transforms/angular-base-package/processors/disambiguateDocPaths.js
+ path = path.gsub(/[A-Z_]/) {|s| s.downcase + '_'}
+ super
+ end
+ include Docs::Angular::Common
+ end
+
version do
- self.release = '12.0.0'
+ self.release = '14.0.0'
self.base_url = 'https://angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
+ include Docs::Angular::Since12
+ end
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
+ version '13' do
+ self.release = '13.3.8'
+ self.base_url = 'https://v13.angular.io/'
+ include Docs::Angular::Since12
+ end
- include Docs::Angular::Common
+ version '12' do
+ self.release = '12.2.13'
+ self.base_url = 'https://v12.angular.io/'
+ include Docs::Angular::Since12
end
version '11' do
self.release = '11.2.14'
self.base_url = 'https://v11.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '10' do
self.release = '10.2.3'
self.base_url = 'https://v10.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '9' do
self.release = '9.1.12'
self.base_url = 'https://v9.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '8' do
self.release = '8.2.14'
self.base_url = 'https://v8.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '7' do
self.release = '7.2.15'
self.base_url = 'https://v7.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '6' do
self.release = '6.1.10'
self.base_url = 'https://v6.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '5' do
self.release = '5.2.11'
self.base_url = 'https://v5.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
version '4' do
self.release = '4.4.6'
self.base_url = 'https://v4.angular.io/'
- self.root_path = 'docs'
-
- html_filters.push 'angular/clean_html', 'angular/entries'
-
- options[:follow_links] = false
- options[:only_patterns] = [/\Aguide/, /\Atutorial/, /\Aapi/]
- options[:fix_urls_before_parse] = ->(url) do
- url.sub! %r{\Aguide/}, '/guide/'
- url.sub! %r{\Atutorial/}, '/tutorial/'
- url.sub! %r{\Aapi/}, '/api/'
- url.sub! %r{\Agenerated/}, '/generated/'
- url
- end
-
include Docs::Angular::Common
end
diff --git a/lib/docs/scrapers/ansible.rb b/lib/docs/scrapers/ansible.rb
index ff43e90d..7b021b26 100644
--- a/lib/docs/scrapers/ansible.rb
+++ b/lib/docs/scrapers/ansible.rb
@@ -32,6 +32,10 @@ module Docs
/\Aroadmap.*/i,
]
+ version do
+ self.base_url = "https://docs.ansible.com/ansible/latest/"
+ end
+
version '2.11' do
self.release = '2.11.0'
self.base_url = "https://docs.ansible.com/ansible/#{version}/"
diff --git a/lib/docs/scrapers/apache.rb b/lib/docs/scrapers/apache.rb
index 81f1b5e9..1fb31640 100644
--- a/lib/docs/scrapers/apache.rb
+++ b/lib/docs/scrapers/apache.rb
@@ -3,7 +3,7 @@ module Docs
self.name = 'Apache HTTP Server'
self.slug = 'apache_http_server'
self.type = 'apache'
- self.release = '2.4.37'
+ self.release = '2.4.52'
self.base_url = 'https://httpd.apache.org/docs/2.4/en/'
self.links = {
home: 'https://httpd.apache.org/'
diff --git a/lib/docs/scrapers/babel.rb b/lib/docs/scrapers/babel.rb
index 78d52d78..2482f9bc 100644
--- a/lib/docs/scrapers/babel.rb
+++ b/lib/docs/scrapers/babel.rb
@@ -25,12 +25,12 @@ module Docs
]
options[:attribution] = <<-HTML
- © 2020 Sebastian McKenzie
+ © 2014-present Sebastian McKenzie
Licensed under the MIT License.
HTML
version '7' do
- self.release = '7.12.6'
+ self.release = '7.16.4'
end
version '6' do
diff --git a/lib/docs/scrapers/bash.rb b/lib/docs/scrapers/bash.rb
index ab4a7bd9..ef46f017 100644
--- a/lib/docs/scrapers/bash.rb
+++ b/lib/docs/scrapers/bash.rb
@@ -1,5 +1,5 @@
module Docs
- class Bash < UrlScraper
+ class Bash < FileScraper
self.type = 'bash'
self.release = '5.1'
self.base_url = 'https://www.gnu.org/software/bash/manual/html_node'
diff --git a/lib/docs/scrapers/bootstrap.rb b/lib/docs/scrapers/bootstrap.rb
index 92be9d1d..ace019d9 100644
--- a/lib/docs/scrapers/bootstrap.rb
+++ b/lib/docs/scrapers/bootstrap.rb
@@ -17,7 +17,7 @@ module Docs
HTML
version '5' do
- self.release = '5.0'
+ self.release = '5.1'
self.base_url = "https://getbootstrap.com/docs/#{self.release}/"
self.root_path = 'getting-started/introduction/'
diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb
index 45776ef3..b53721c4 100644
--- a/lib/docs/scrapers/chef.rb
+++ b/lib/docs/scrapers/chef.rb
@@ -20,6 +20,36 @@ module Docs
We are not affiliated with, endorsed or sponsored by Chef Inc.
HTML
+ version '18' do
+ self.release = '18.0.15'
+
+ options[:container] = '.off-canvas-wrapper'
+
+ options[:skip] = [
+ '/automate/api/',
+ '/habitat/supervisor_api/',
+ '/habitat/builder_api/'
+ ]
+
+ html_filters.push 'chef/entries', 'chef/clean_html'
+
+ end
+
+ version '17' do
+ self.release = '17.9.18'
+
+ options[:container] = '.off-canvas-wrapper'
+
+ options[:skip] = [
+ '/automate/api/',
+ '/habitat/supervisor_api/',
+ '/habitat/builder_api/'
+ ]
+
+ html_filters.push 'chef/entries', 'chef/clean_html'
+
+ end
+
version '16' do
self.release = '16.8.14'
@@ -82,8 +112,8 @@ module Docs
end
def get_latest_version(opts)
- doc = fetch_doc('https://downloads.chef.io/products/infra', opts)
- doc.at_css('#versions > option').content.strip
+ tags = get_github_tags('chef', 'chef', opts)
+ tags[0]['name'][1..-1]
end
end
diff --git a/lib/docs/scrapers/clojure.rb b/lib/docs/scrapers/clojure.rb
index 3a27d097..50146a68 100644
--- a/lib/docs/scrapers/clojure.rb
+++ b/lib/docs/scrapers/clojure.rb
@@ -17,11 +17,16 @@ module Docs
Licensed under the Eclipse Public License 1.0.
HTML
- version '1.10' do
- self.release = '1.10 (stable)'
+ version '1.11' do
+ self.release = '1.11'
self.base_url = 'https://clojure.github.io/clojure/'
end
+ version '1.10' do
+ self.release = '1.10.3'
+ self.base_url = "https://clojure.github.io/clojure/branch-clojure-#{self.release}/"
+ end
+
version '1.9' do
self.release = '1.9 (legacy)'
self.base_url = 'https://clojure.github.io/clojure/branch-clojure-1.9.0/'
diff --git a/lib/docs/scrapers/cmake.rb b/lib/docs/scrapers/cmake.rb
index 69a8c2f6..dc79429a 100644
--- a/lib/docs/scrapers/cmake.rb
+++ b/lib/docs/scrapers/cmake.rb
@@ -13,13 +13,18 @@ module Docs
options[:title] = false
options[:root_title] = 'CMake Reference Documentation'
options[:skip] = %w(release/index.html genindex.html search.html)
- options[:skip_patterns] = [/\Agenerator/, /\Acpack_gen/, /\Ainclude/, /\Arelease/]
+ options[:skip_patterns] = [/\Agenerator/, /\Acpack_gen/, /\Ainclude/, /\Arelease/, /tutorial\/(\w*%20)+/]
options[:attribution] = <<-HTML
© 2000–2021 Kitware, Inc. and Contributors
Licensed under the BSD 3-clause License.
HTML
+ version '3.21' do
+ self.release = '3.21'
+ self.base_url = "https://cmake.org/cmake/help/v#{self.version}/"
+ end
+
version '3.20' do
self.release = '3.20'
self.base_url = "https://cmake.org/cmake/help/v#{self.version}/"
diff --git a/lib/docs/scrapers/codeception.rb b/lib/docs/scrapers/codeception.rb
index 920080b0..64ebc263 100644
--- a/lib/docs/scrapers/codeception.rb
+++ b/lib/docs/scrapers/codeception.rb
@@ -2,7 +2,7 @@ module Docs
class Codeception < UrlScraper
self.name = 'Codeception'
self.type = 'codeception'
- self.release = '4.1.12'
+ self.release = '4.1.22'
self.base_url = 'https://codeception.com/docs/'
self.root_path = 'index.html'
self.links = {
@@ -20,8 +20,7 @@ module Docs
HTML
def get_latest_version(opts)
- doc = fetch_doc('https://codeception.com/changelog', opts)
- doc.at_css('#page > h4').content
+ get_github_tags("Codeception", "Codeception", opts)[1]["name"]
end
end
end
diff --git a/lib/docs/scrapers/codeigniter.rb b/lib/docs/scrapers/codeigniter.rb
index d093be3e..550f1ceb 100644
--- a/lib/docs/scrapers/codeigniter.rb
+++ b/lib/docs/scrapers/codeigniter.rb
@@ -29,13 +29,13 @@ module Docs
]
options[:attribution] = <<-HTML
- © 2014–2020 British Columbia Institute of Technology
+ © 2014–2021 British Columbia Institute of Technology
Licensed under the MIT License.
HTML
version '4' do
- self.release = '4.0.4'
- self.base_url = 'https://codeigniter.com/userguide4/'
+ self.release = '4.1.5'
+ self.base_url = 'https://codeigniter.com/user_guide/'
options[:container] = '.document > div'
end
diff --git a/lib/docs/scrapers/coffeescript.rb b/lib/docs/scrapers/coffeescript.rb
index a57abd52..5282f676 100644
--- a/lib/docs/scrapers/coffeescript.rb
+++ b/lib/docs/scrapers/coffeescript.rb
@@ -11,12 +11,12 @@ module Docs
options[:skip_links] = true
options[:attribution] = <<-HTML
- © 2009–2020 Jeremy Ashkenas
+ © 2009–2022 Jeremy Ashkenas
Licensed under the MIT License.
HTML
version '2' do
- self.release = '2.5.1'
+ self.release = '2.7.0'
self.base_url = 'https://coffeescript.org/'
html_filters.push 'coffeescript/entries', 'coffeescript/clean_html', 'title'
diff --git a/lib/docs/scrapers/crystal.rb b/lib/docs/scrapers/crystal.rb
index 8dd794ef..3105aacc 100644
--- a/lib/docs/scrapers/crystal.rb
+++ b/lib/docs/scrapers/crystal.rb
@@ -1,8 +1,17 @@
module Docs
class Crystal < UrlScraper
+ include MultipleBaseUrls
self.type = 'crystal'
- self.base_url = 'https://crystal-lang.org/'
- self.initial_paths = %w(reference/index.html)
+ self.release = '1.4.0'
+ self.base_urls = [
+ "https://crystal-lang.org/api/#{release}/",
+ "https://crystal-lang.org/reference/#{release[0..2]}/",
+ ]
+ def initial_urls
+ [ "https://crystal-lang.org/api/#{self.class.release}/index.html",
+ "https://crystal-lang.org/reference/#{self.class.release[0..2]}/index.html" ]
+ end
+
self.links = {
home: 'https://crystal-lang.org/',
code: 'https://github.com/crystal-lang/crystal'
@@ -11,7 +20,7 @@ module Docs
html_filters.push 'crystal/entries', 'crystal/clean_html'
options[:attribution] = ->(filter) {
- if filter.slug.start_with?('reference')
+ if filter.current_url.path.start_with?('/reference/')
<<-HTML
To the extent possible under law, the persons who contributed to this work
have waived
all copyright and related or neighboring rights to this work
@@ -19,23 +28,12 @@ module Docs
HTML
else
<<-HTML
- © 2012–2021 Manas Technology Solutions.
+ © 2012–2022 Manas Technology Solutions.
Licensed under the Apache License, Version 2.0.
HTML
end
}
- self.release = '1.0.0'
- self.root_path = "api/#{release}/index.html"
-
- options[:only_patterns] = [/\Aapi\/#{release}\//, /\Areference\//]
- options[:skip_patterns] = [/debug/i]
-
- options[:replace_paths] = {
- "api/#{release}/" => "api/#{release}/index.html",
- 'reference/' => 'reference/index.html'
- }
-
def get_latest_version(opts)
doc = fetch_doc('https://crystal-lang.org/', opts)
doc.at_css('.latest-release').content.scan(/([0-9.]+)/)[0][0]
diff --git a/lib/docs/scrapers/cypress.rb b/lib/docs/scrapers/cypress.rb
index 566449f8..94c83ca0 100644
--- a/lib/docs/scrapers/cypress.rb
+++ b/lib/docs/scrapers/cypress.rb
@@ -4,7 +4,7 @@ module Docs
class Cypress < UrlScraper
self.name = 'Cypress'
self.type = 'cypress'
- self.release = '7.2.0'
+ self.release = '9.4.1'
self.base_url = 'https://docs.cypress.io'
self.root_path = '/api/table-of-contents.html'
self.links = {
diff --git a/lib/docs/scrapers/d3.rb b/lib/docs/scrapers/d3.rb
index ef3c8e49..c31b24e1 100644
--- a/lib/docs/scrapers/d3.rb
+++ b/lib/docs/scrapers/d3.rb
@@ -12,10 +12,26 @@ module Docs
options[:container] = '.markdown-body'
options[:attribution] = <<-HTML
- © 2010–2020 Michael Bostock
+ © 2010–2022 Michael Bostock
Licensed under the BSD License.
HTML
+ version '7' do
+ self.release = '7.4.0'
+ self.base_url = 'https://github.com/d3/'
+ self.root_path = 'd3/blob/master/API.md'
+
+ html_filters.push 'd3/clean_html', 'd3/entries_v4'
+
+ options[:only_patterns] = [/\Ad3[\-\w]+\z/, /\Ad3\/blob\/master\/changes\.md\z/i]
+ options[:skip_patterns] = [/3\.x-api-reference/]
+
+ options[:fix_urls] = ->(url) do
+ url.sub! %r{/blob/master/readme.md}i, ''
+ url
+ end
+ end
+
version '6' do
self.release = '6.7.0'
self.base_url = 'https://github.com/d3/'
diff --git a/lib/docs/scrapers/dart.rb b/lib/docs/scrapers/dart.rb
index a7b92d9d..c12d051a 100644
--- a/lib/docs/scrapers/dart.rb
+++ b/lib/docs/scrapers/dart.rb
@@ -21,7 +21,7 @@ module Docs
HTML
version '2' do
- self.release = '2.13.0'
+ self.release = '2.17.0'
self.base_url = "https://api.dart.dev/stable/#{release}/"
end
diff --git a/lib/docs/scrapers/deno.rb b/lib/docs/scrapers/deno.rb
new file mode 100644
index 00000000..4290878b
--- /dev/null
+++ b/lib/docs/scrapers/deno.rb
@@ -0,0 +1,23 @@
+module Docs
+ class Deno < UrlScraper
+ self.name = 'Deno'
+ self.type = 'simple'
+ self.release = '1.22.0'
+ self.base_url = 'https://doc.deno.land/deno/stable/'
+ self.links = {
+ home: 'https://deno.land/',
+ code: 'https://github.com/denoland/deno'
+ }
+
+ html_filters.push 'deno/entries', 'deno/clean_html'
+
+ # https://github.com/denoland/manual/blob/main/LICENSE
+ options[:attribution] = <<-HTML
+ © 2018–2022 the Deno authors
+ HTML
+
+ def get_latest_version(opts)
+ get_latest_github_release('denoland', 'deno', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/django.rb b/lib/docs/scrapers/django.rb
index 11cf81bd..d22e2083 100644
--- a/lib/docs/scrapers/django.rb
+++ b/lib/docs/scrapers/django.rb
@@ -34,6 +34,11 @@ module Docs
Licensed under the BSD License.
HTML
+ version '4.0' do
+ self.release = '4.0'
+ self.base_url = 'https://docs.djangoproject.com/en/4.0/'
+ end
+
version '3.2' do
self.release = '3.2'
self.base_url = 'https://docs.djangoproject.com/en/3.2/'
diff --git a/lib/docs/scrapers/eigen3.rb b/lib/docs/scrapers/eigen3.rb
new file mode 100644
index 00000000..c687debe
--- /dev/null
+++ b/lib/docs/scrapers/eigen3.rb
@@ -0,0 +1,76 @@
+module Docs
+ class Eigen3 < UrlScraper
+ self.name = 'Eigen3'
+ self.type = 'eigen3'
+ self.slug = 'eigen3'
+ self.base_url = 'https://eigen.tuxfamily.org/dox/'
+ self.root_path = 'index.html'
+ self.initial_paths = [
+ "modules.html"
+ ]
+ self.release = '3.4.0'
+
+ self.links = {
+ home: 'https://eigen.tuxfamily.org',
+ code: 'https://gitlab.com/libeigen/eigen'
+ }
+
+ html_filters.push 'eigen3/entries', 'eigen3/clean_html', 'title'
+
+ # Remove the `clean_text` because Doxygen are actually creating empty
+ # anchor such as to do anchor link.. and that anchor
+ # will be removed by clean_text
+ self.text_filters = FilterStack.new
+ text_filters.push 'images', 'inner_html', 'attribution'
+
+
+
+ def get_latest_version(opts)
+ tags = get_gitlab_tags("gitlab.com", "libeigen", "eigen", opts)
+ tags[0]['name']
+ end
+
+ options[:attribution] = <<-HTML
+ © Eigen.
+ Licensed under the MPL2 License.
+ HTML
+
+ # Skip source code since it doesn't provide any useful docs
+ options[:skip_patterns] = [/_source/, /-members/, /__Reference\.html/, /_chapter\.html/, /\.txt/, /\.tgz/]
+
+ # TODO: replace cppreference
+ # options[:replace_urls] = { 'http://en.cppreference.com/w/cpp/' => 'cpp/' }
+
+ def parse(response) # Hook here because Nokogori removes whitespace from code fragments
+ last_idx = 0
+ # Process nested s inside code fragment div.
+ while not (last_idx = response.body.index('', last_idx)).nil?
+ # enter code fragment
+ level = 1
+ while not (last_idx = response.body.index(/<\/?div/, last_idx+1)).nil?
+ # skip nested divs inside.
+ if response.body[last_idx..last_idx+3] == ''
+ response.body[last_idx..last_idx+5] = ''
+ end
+ end
+ response.body.gsub! /[\r\n\s]*]*>.*<\/div>[\r\n\s]*/, ""
+ response.body.gsub! /(.*?)<\/div>/m, "\\1"
+ response.body.gsub! '', ''
+ super
+ end
+
+ def process_response?(response)
+ return false unless super
+ # Remove Empty pages.
+ response.body.index(/[\r\n\s]*<\/div>/m).nil? and \
+ response.body.index(/TODO: write this dox page!<\/p>/).nil?
+ end
+ end
+end
diff --git a/lib/docs/scrapers/electron.rb b/lib/docs/scrapers/electron.rb
index da105f39..2af9dfcc 100644
--- a/lib/docs/scrapers/electron.rb
+++ b/lib/docs/scrapers/electron.rb
@@ -1,8 +1,8 @@
module Docs
class Electron < UrlScraper
self.type = 'simple'
- self.base_url = 'https://www.electronjs.org/docs'
- self.release = '12.0.0'
+ self.base_url = 'https://www.electronjs.org/docs/latest'
+ self.release = '18.0.1'
self.links = {
home: 'https://www.electronjs.org/',
code: 'https://github.com/electron/electron'
@@ -13,7 +13,9 @@ module Docs
options[:trailing_slash] = false
options[:container] = 'main'
options[:skip] = %w(guides development tutorial versions all)
- options[:skip_patterns] = [/\/history\z/]
+ options[:skip_patterns] = [
+ /\/history\z/,
+ ]
options[:replace_paths] = {
'api/web-view-tag' => 'api/webview-tag'
}
@@ -24,8 +26,8 @@ module Docs
HTML
def get_latest_version(opts)
- doc = fetch_doc('https://www.electronjs.org/docs', opts)
- doc.at_css('.docs-version').content
+ doc = fetch_doc('https://www.electronjs.org/releases/stable', opts)
+ doc.at_css(".tag").content.gsub!(/[a-zA-Z]/, '')
end
end
end
diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb
index 26fd00e3..3b423264 100644
--- a/lib/docs/scrapers/elixir.rb
+++ b/lib/docs/scrapers/elixir.rb
@@ -33,6 +33,19 @@ module Docs
"https://elixir-lang.org/getting-started/introduction.html" ]
end
+ version '1.13' do
+ self.release = '1.13.4'
+ self.base_urls = [
+ "https://hexdocs.pm/elixir/#{release}/",
+ "https://hexdocs.pm/eex/#{release}/",
+ "https://hexdocs.pm/ex_unit/#{release}/",
+ "https://hexdocs.pm/iex/#{release}/",
+ "https://hexdocs.pm/logger/#{release}/",
+ "https://hexdocs.pm/mix/#{release}/",
+ 'https://elixir-lang.org/getting-started/'
+ ]
+ end
+
version '1.12' do
self.release = '1.12.0'
self.base_urls = [
diff --git a/lib/docs/scrapers/esbuild.rb b/lib/docs/scrapers/esbuild.rb
new file mode 100644
index 00000000..5484f155
--- /dev/null
+++ b/lib/docs/scrapers/esbuild.rb
@@ -0,0 +1,27 @@
+module Docs
+ class Esbuild < UrlScraper
+ self.name = 'esbuild'
+ self.slug = 'esbuild'
+ self.type = 'simple'
+ self.links = {
+ home: 'https://esbuild.github.io/',
+ code: 'https://github.com/evanw/esbuild'
+ }
+
+ options[:container] = 'main'
+ options[:root_title] = 'esbuild'
+
+ options[:attribution] = <<-HTML
+ © 2020 Evan Wallace
+ Licensed under the MIT License.
+ HTML
+
+ self.release = '0.14.2'
+ self.base_url = 'https://esbuild.github.io/'
+ html_filters.push 'esbuild/clean_html', 'esbuild/entries'
+
+ def get_latest_version(opts)
+ get_npm_version('esbuild', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/eslint.rb b/lib/docs/scrapers/eslint.rb
index 86bd1f51..4210523b 100644
--- a/lib/docs/scrapers/eslint.rb
+++ b/lib/docs/scrapers/eslint.rb
@@ -2,7 +2,7 @@ module Docs
class Eslint < UrlScraper
self.name = 'ESLint'
self.type = 'simple'
- self.release = '7.26.0'
+ self.release = '8.16.0'
self.base_url = 'https://eslint.org/docs/'
self.root_path = 'user-guide/getting-started'
self.links = {
@@ -17,7 +17,7 @@ module Docs
options[:replace_paths] = { 'user-guide' => 'user-guide/' }
options[:attribution] = <<-HTML
- © JS Foundation and other contributors
+ © OpenJS Foundation and other contributors
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/fish.rb b/lib/docs/scrapers/fish.rb
index 08f240e9..4202a30d 100644
--- a/lib/docs/scrapers/fish.rb
+++ b/lib/docs/scrapers/fish.rb
@@ -12,10 +12,26 @@ module Docs
# https://fishshell.com/docs/current/license.html
options[:attribution] = <<-HTML
- © 2020 fish-shell developers
+ © 2022 fish-shell developers
Licensed under the GNU General Public License, version 2.
HTML
+ version '3.4' do
+ self.release = '3.4.0'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
+ version '3.3' do
+ self.release = '3.3.0'
+ self.base_url = "https://fishshell.com/docs/#{version}/"
+
+ options[:skip].concat %w(genindex.html relnotes.html)
+ html_filters.push 'sphinx/clean_html', 'fish/clean_html_sphinx', 'fish/entries_sphinx'
+ end
+
version '3.2' do
self.release = '3.2.0'
self.base_url = "https://fishshell.com/docs/#{version}/"
diff --git a/lib/docs/scrapers/flask.rb b/lib/docs/scrapers/flask.rb
index 4251a61b..7b644b98 100755
--- a/lib/docs/scrapers/flask.rb
+++ b/lib/docs/scrapers/flask.rb
@@ -9,15 +9,20 @@ module Docs
html_filters.push 'flask/entries', 'sphinx/clean_html'
- options[:container] = '.body > .section'
+ options[:container] = '.body > section'
options[:skip] = %w(extensiondev/ styleguide/ upgrading/ changelog/ license/ contributing/)
options[:skip_patterns] = [/\Atutorial\//]
options[:attribution] = <<-HTML
- © 2007–2021 Pallets
+ © 2007–2022 Pallets
Licensed under the BSD 3-clause License.
HTML
+ version '2.1' do
+ self.release = '2.1.x'
+ self.base_url = "https://flask.palletsprojects.com/en/#{self.release}/"
+ end
+
version '2.0' do
self.release = '2.0.x'
self.base_url = "https://flask.palletsprojects.com/en/#{self.release}/"
diff --git a/lib/docs/scrapers/flow.rb b/lib/docs/scrapers/flow.rb
index 8158ac7d..d86c7563 100644
--- a/lib/docs/scrapers/flow.rb
+++ b/lib/docs/scrapers/flow.rb
@@ -1,7 +1,7 @@
module Docs
class Flow < UrlScraper
self.type = 'simple'
- self.release = '0.148.0'
+ self.release = '0.179.0'
self.base_url = 'https://flow.org/en/docs/'
self.links = {
home: 'https://flow.org/',
diff --git a/lib/docs/scrapers/git.rb b/lib/docs/scrapers/git.rb
index bc62ff5c..aaebcef9 100644
--- a/lib/docs/scrapers/git.rb
+++ b/lib/docs/scrapers/git.rb
@@ -1,7 +1,7 @@
module Docs
class Git < UrlScraper
self.type = 'git'
- self.release = '2.31.0'
+ self.release = '2.36.0'
self.base_url = 'https://git-scm.com/docs'
self.initial_paths = %w(/git.html)
self.links = {
@@ -16,7 +16,7 @@ module Docs
options[:skip] = %w(/howto-index.html)
options[:attribution] = <<-HTML
- © 2012–2021 Scott Chacon and others
+ © 2012–2022 Scott Chacon and others
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/gnu/gcc.rb b/lib/docs/scrapers/gnu/gcc.rb
index 7de85af0..873ee13a 100644
--- a/lib/docs/scrapers/gnu/gcc.rb
+++ b/lib/docs/scrapers/gnu/gcc.rb
@@ -46,6 +46,16 @@ module Docs
'Wtrigraphs.html' => 'Invocation.html'
}
+ version '12' do
+ self.release = '12.1.0'
+ self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
+ end
+
+ version '12 CPP' do
+ self.release = '12.1.0'
+ self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
+ end
+
version '11' do
self.release = '11.1.0'
self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
diff --git a/lib/docs/scrapers/gnu/gnu_fortran.rb b/lib/docs/scrapers/gnu/gnu_fortran.rb
index 6eec43ab..72c516f9 100644
--- a/lib/docs/scrapers/gnu/gnu_fortran.rb
+++ b/lib/docs/scrapers/gnu/gnu_fortran.rb
@@ -6,6 +6,11 @@ module Docs
home: 'https://gcc.gnu.org/fortran/'
}
+ version '12' do
+ self.release = '12.1.0'
+ self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
+ end
+
version '11' do
self.release = '11.1.0'
self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
diff --git a/lib/docs/scrapers/gnu_make.rb b/lib/docs/scrapers/gnu_make.rb
new file mode 100644
index 00000000..a2837a46
--- /dev/null
+++ b/lib/docs/scrapers/gnu_make.rb
@@ -0,0 +1,34 @@
+# coding: utf-8
+module Docs
+ class GnuMake < FileScraper
+ self.name = 'GNU Make'
+ self.type = 'gnu_make'
+ self.slug = 'gnu_make'
+ self.release = '4.3'
+ self.base_url= 'https://www.gnu.org/software/make/manual/html_node/'
+ self.root_path = 'index.html'
+ self.links = {
+ home:'https://www.gnu.org/software/make/manual/html_node/',
+ code: 'http://git.savannah.gnu.org/cgit/make.git/'
+ }
+
+ html_filters.push 'gnu_make/entries', 'gnu_make/clean_html'
+
+ options[:skip] = [
+ 'Concept-Index.html',
+ 'Name-Index.html',
+ 'GNU-Free-Documentation-License.html'
+ ]
+
+ options[:attribution]= <<-HTML
+ Copyright © 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Free Software Foundation, Inc.
+ Licensed under the GNU Free Documentation License.
+ HTML
+
+ def get_latest_version(opts)
+ body = fetch("https://www.gnu.org/software/make/manual/html_node/", opts)
+ body.scan(/version \d*\.?\d*/)[0].sub('version', '')
+ end
+
+ end
+end
diff --git a/lib/docs/scrapers/go.rb b/lib/docs/scrapers/go.rb
index 18abcbac..3b2e5511 100644
--- a/lib/docs/scrapers/go.rb
+++ b/lib/docs/scrapers/go.rb
@@ -1,14 +1,24 @@
module Docs
class Go < UrlScraper
self.type = 'go'
- self.release = '1.16'
+ self.release = '1.18.0'
self.base_url = 'https://golang.org/pkg/'
self.links = {
home: 'https://golang.org/',
code: 'https://go.googlesource.com/go'
}
+ # Run godoc locally, since https://golang.org/pkg/ redirects to https://pkg.go.dev/std with rate limiting / scraping protection.
+
+ # docker run --expose=6060 --rm -it docker.io/golang:1.18.0
+ #docker# go install golang.org/x/tools/cmd/godoc@latest
+ #docker# rm -r /usr/local/go/test/
+ #docker# godoc -http 0.0.0.0:6060 -v
+ self.base_url = 'http://localhost:6060/pkg/'
+
+ html_filters.push 'clean_local_urls'
html_filters.push 'go/clean_html', 'go/entries'
+ text_filters.replace 'attribution', 'go/attribution'
options[:trailing_slash] = true
options[:container] = '#page .container'
@@ -16,7 +26,7 @@ module Docs
options[:skip_patterns] = [/\/\//]
options[:fix_urls] = ->(url) do
- url.sub 'https://golang.org/pkg//', 'https://golang.org/pkg/'
+ url.sub '/pkg//', '/pkg/'
end
options[:attribution] = <<-HTML
@@ -25,8 +35,8 @@ module Docs
HTML
def get_latest_version(opts)
- doc = fetch_doc('https://golang.org/project/', opts)
- doc.at_css('#page ul > li > a').text[3..-1]
+ doc = fetch_doc('https://go.dev/dl/', opts)
+ doc.at_css('.download[href]')['href'][/go1[0-9.]+[0-9]/][2..]
end
private
diff --git a/lib/docs/scrapers/groovy.rb b/lib/docs/scrapers/groovy.rb
index ebd4242c..37197372 100755
--- a/lib/docs/scrapers/groovy.rb
+++ b/lib/docs/scrapers/groovy.rb
@@ -19,12 +19,17 @@ module Docs
]
options[:attribution] = <<-HTML
- © 2003-2020 The Apache Software Foundation
+ © 2003-2022 The Apache Software Foundation
Licensed under the Apache license.
HTML
+ version '4.0' do
+ self.release = '4.0.0'
+ self.base_url = "https://docs.groovy-lang.org/#{self.release}/html/gapi/"
+ end
+
version '3.0' do
- self.release = '3.0.7'
+ self.release = '3.0.9'
self.base_url = "https://docs.groovy-lang.org/#{self.release}/html/gapi/"
end
diff --git a/lib/docs/scrapers/haproxy.rb b/lib/docs/scrapers/haproxy.rb
index 6c354787..db558563 100644
--- a/lib/docs/scrapers/haproxy.rb
+++ b/lib/docs/scrapers/haproxy.rb
@@ -6,7 +6,7 @@ module Docs
self.initial_paths = %w(intro.html configuration.html management.html)
self.links = {
home: 'https://www.haproxy.org/',
- code: 'https://github.com/haproxy/haproxy/'
+ code: 'https://github.com/haproxy/haproxy'
}
html_filters.push 'haproxy/clean_html', 'haproxy/entries'
@@ -20,6 +20,11 @@ module Docs
Licensed under the GNU General Public License version 2.
HTML
+ version '2.5' do
+ self.release = '2.5.0'
+ self.base_url = "https://cbonte.github.io/haproxy-dconv/#{self.version}/"
+ end
+
version '2.4' do
self.release = '2.4.0'
self.base_url = "https://cbonte.github.io/haproxy-dconv/#{self.version}/"
diff --git a/lib/docs/scrapers/haskell.rb b/lib/docs/scrapers/haskell.rb
index 5ed6e967..471a7e0d 100755
--- a/lib/docs/scrapers/haskell.rb
+++ b/lib/docs/scrapers/haskell.rb
@@ -59,7 +59,7 @@ module Docs
end
version '9' do
- self.release = '9.0.1'
+ self.release = '9.2.1'
self.base_url = "https://downloads.haskell.org/~ghc/#{release}/docs/html/"
options[:container] = ->(filter) {filter.subpath.start_with?('users_guide') ? '.document' : '#content'}
end
diff --git a/lib/docs/scrapers/homebrew.rb b/lib/docs/scrapers/homebrew.rb
index 6a7b5aed..b6658701 100644
--- a/lib/docs/scrapers/homebrew.rb
+++ b/lib/docs/scrapers/homebrew.rb
@@ -2,7 +2,7 @@ module Docs
class Homebrew < UrlScraper
self.name = 'Homebrew'
self.type = 'simple'
- self.release = '3.1.2'
+ self.release = '3.3.11'
self.base_url = 'https://docs.brew.sh/'
self.links = {
home: 'https://brew.sh',
diff --git a/lib/docs/scrapers/http.rb b/lib/docs/scrapers/http.rb
index 9d989c6a..f1fb5ab5 100644
--- a/lib/docs/scrapers/http.rb
+++ b/lib/docs/scrapers/http.rb
@@ -2,15 +2,25 @@ module Docs
class Http < Mdn
include MultipleBaseUrls
+ # release = '2021-10-22'
self.name = 'HTTP'
- self.base_urls = ['https://developer.mozilla.org/en-US/docs/Web/HTTP', 'https://tools.ietf.org/html/']
+ self.base_urls = [
+ 'https://developer.mozilla.org/en-US/docs/Web/HTTP',
+ 'https://datatracker.ietf.org/doc/html/',
+ ]
html_filters.push 'http/clean_html', 'http/entries', 'title'
options[:root_title] = 'HTTP'
- options[:title] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? false : filter.default_title }
- options[:container] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? '.content' : nil }
- options[:skip_links] = ->(filter) { filter.current_url.host == 'tools.ietf.org' ? true : false }
+ options[:title] = ->(filter) do
+ filter.current_url.host == 'datatracker.ietf.org' ? false : filter.default_title
+ end
+ options[:container] = ->(filter) do
+ filter.current_url.host == 'datatracker.ietf.org' ? '.content .draftcontent' : Docs::Mdn.options[:container]
+ end
+ options[:skip_links] = ->(filter) do
+ filter.current_url.host == 'datatracker.ietf.org' ? true : false
+ end
options[:replace_paths] = { '/Access_control_CORS' => '/CORS' }
options[:fix_urls] = ->(url) do
url.sub! %r{(Status/\d\d\d)_[A-Z].+}, '\1'
@@ -18,7 +28,7 @@ module Docs
end
options[:attribution] = ->(filter) do
- if filter.current_url.host == 'tools.ietf.org'
+ if filter.current_url.host == 'datatracker.ietf.org'
"© document authors. All rights reserved."
else
Docs::Mdn.options[:attribution]
@@ -27,16 +37,16 @@ module Docs
def initial_urls
%w(https://developer.mozilla.org/en-US/docs/Web/HTTP
- https://tools.ietf.org/html/rfc2616
- https://tools.ietf.org/html/rfc4918
- https://tools.ietf.org/html/rfc7230
- https://tools.ietf.org/html/rfc7231
- https://tools.ietf.org/html/rfc7232
- https://tools.ietf.org/html/rfc7233
- https://tools.ietf.org/html/rfc7234
- https://tools.ietf.org/html/rfc7235
- https://tools.ietf.org/html/rfc7540
- https://tools.ietf.org/html/rfc5023)
+ https://datatracker.ietf.org/doc/html/rfc2616
+ https://datatracker.ietf.org/doc/html/rfc4918
+ https://datatracker.ietf.org/doc/html/rfc7230
+ https://datatracker.ietf.org/doc/html/rfc7231
+ https://datatracker.ietf.org/doc/html/rfc7232
+ https://datatracker.ietf.org/doc/html/rfc7233
+ https://datatracker.ietf.org/doc/html/rfc7234
+ https://datatracker.ietf.org/doc/html/rfc7235
+ https://datatracker.ietf.org/doc/html/rfc7540
+ https://datatracker.ietf.org/doc/html/rfc5023)
end
end
end
diff --git a/lib/docs/scrapers/i3.rb b/lib/docs/scrapers/i3.rb
new file mode 100644
index 00000000..abc4e791
--- /dev/null
+++ b/lib/docs/scrapers/i3.rb
@@ -0,0 +1,28 @@
+module Docs
+ class I3 < UrlScraper
+ self.name = 'i3'
+ self.type = 'simple'
+ self.slug = 'i3'
+ self.release = '4.20.1'
+ self.base_url = 'https://i3wm.org/docs/userguide.html'
+ self.links = {
+ home: 'https://i3wm.org/',
+ code: 'https://github.com/i3/i3'
+ }
+
+ html_filters.push 'i3/entries', 'title'
+
+ options[:container] = 'main'
+ options[:skip_links] = true
+
+ options[:attribution] = <<-HTML
+ © 2009, Michael Stapelberg and contributors
+ HTML
+
+ def get_latest_version(opts)
+ tags = get_github_tags('i3', 'i3', opts)
+ tag = tags.find {|tag| tag['name'].start_with?('4.')}
+ tag['name']
+ end
+ end
+end
diff --git a/lib/docs/scrapers/immutable.rb b/lib/docs/scrapers/immutable.rb
index 8b1b47a2..1768ac66 100644
--- a/lib/docs/scrapers/immutable.rb
+++ b/lib/docs/scrapers/immutable.rb
@@ -3,58 +3,31 @@ module Docs
self.name = 'Immutable.js'
self.slug = 'immutable'
self.type = 'simple'
- self.release = '3.8.1'
- self.base_url = 'https://facebook.github.io/immutable-js/docs/'
+ self.release = '4.0.0'
+ self.base_url = 'https://immutable-js.com/docs/v4.0.0/'
self.links = {
- home: 'https://facebook.github.io/immutable-js/',
+ home: 'https://immutable-js.com/',
code: 'https://github.com/facebook/immutable-js'
}
- html_filters.push 'immutable/clean_html', 'immutable/entries', 'title'
+ html_filters.push 'immutable/clean_html', 'immutable/entries'
- options[:skip_links] = true
options[:container] = '.docContents'
options[:root_title] = 'Immutable.js'
+ options[:trailing_slash] = true
+ options[:fix_urls] = ->(url) do
+ url.sub! '/index.html', ''
+ url.sub! '/index', ''
+ url
+ end
+
+
options[:attribution] = <<-HTML
- © 2014–2015 Facebook, Inc.
+ © 2014–present, Lee Byron and other contributors
Licensed under the 3-clause BSD License.
HTML
- stub '' do
- capybara = load_capybara_selenium
- capybara.app_host = 'https://facebook.github.io'
- capybara.visit(URL.parse(self.base_url).path)
- capybara.execute_script <<-JS
- var content, event, links, link;
-
- event = document.createEvent('Event');
- event.initEvent('hashchange', false, false);
-
- content = document.querySelector('.docContents section').cloneNode(true);
- links = Array.prototype.slice.call(document.querySelectorAll('.sideBar .scrollContent a'));
-
- while (link = links.shift()) {
- if (!document.body.contains(link)) {
- document.body.appendChild(link);
- }
-
- link.click();
- dispatchEvent(event);
- content.innerHTML += document.querySelector('.docContents').innerHTML;
-
- document.querySelectorAll('.sideBar .scrollContent .groupTitle').forEach(function(el) {
- if (el.textContent == 'Types') {
- Array.prototype.unshift.apply(links, Array.prototype.slice.call(el.parentNode.querySelectorAll('a')));
- }
- });
- }
-
- document.querySelector('.docContents').innerHTML = content.innerHTML;
- JS
- capybara.html
- end
-
def get_latest_version(opts)
get_npm_version('immutable', opts)
end
diff --git a/lib/docs/scrapers/jasmine.rb b/lib/docs/scrapers/jasmine.rb
index bdcb3626..753e6134 100644
--- a/lib/docs/scrapers/jasmine.rb
+++ b/lib/docs/scrapers/jasmine.rb
@@ -1,8 +1,8 @@
module Docs
class Jasmine < UrlScraper
- self.type = 'simple'
- self.release = '3.7.1'
- self.base_url = 'https://jasmine.github.io/api/3.7/'
+ self.type = 'jasmine'
+ self.release = '4.0.0'
+ self.base_url = 'https://jasmine.github.io/api/4.0/'
self.root_path = 'index.html'
self.links = {
home: 'https://jasmine.github.io/',
diff --git a/lib/docs/scrapers/jest.rb b/lib/docs/scrapers/jest.rb
index 50fb3397..619a3d6a 100644
--- a/lib/docs/scrapers/jest.rb
+++ b/lib/docs/scrapers/jest.rb
@@ -1,8 +1,14 @@
module Docs
class Jest < UrlScraper
+ include MultipleBaseUrls
self.type = 'simple'
- self.release = '26.6'
- self.base_url = 'https://jestjs.io/docs/en/'
+ self.release = '28.0.3'
+
+ self.base_urls = [
+ 'https://jestjs.io/docs/',
+ 'https://jestjs.io/docs/expect'
+ ]
+
self.root_path = 'getting-started'
self.links = {
home: 'https://jestjs.io/',
@@ -11,10 +17,13 @@ module Docs
html_filters.push 'jest/entries', 'jest/clean_html'
- options[:container] = '.docMainWrapper'
+ options[:skip_patterns] = [
+ /^next/,
+ /\d+\.[x\d]/ # avoid deprecated versions
+ ]
options[:attribution] = <<-HTML
- © 2020 Facebook, Inc.
+ © 2022 Facebook, Inc.
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/jquery/jquery_ui.rb b/lib/docs/scrapers/jquery/jquery_ui.rb
index 021d1d22..d4f66fef 100644
--- a/lib/docs/scrapers/jquery/jquery_ui.rb
+++ b/lib/docs/scrapers/jquery/jquery_ui.rb
@@ -2,7 +2,7 @@ module Docs
class JqueryUi < Jquery
self.name = 'jQuery UI'
self.slug = 'jqueryui'
- self.release = '1.12.1'
+ self.release = '1.13.0'
self.base_url = 'https://api.jqueryui.com'
self.root_path = '/category/all'
diff --git a/lib/docs/scrapers/jsdoc.rb b/lib/docs/scrapers/jsdoc.rb
index 32799e8c..49745d60 100644
--- a/lib/docs/scrapers/jsdoc.rb
+++ b/lib/docs/scrapers/jsdoc.rb
@@ -2,7 +2,7 @@ module Docs
class Jsdoc < UrlScraper
self.name = 'JSDoc'
self.type = 'simple'
- self.release = '3.5.5'
+ self.release = '3.6.7'
self.base_url = 'https://jsdoc.app/'
self.root_path = 'index.html'
self.links = {
@@ -23,7 +23,7 @@ module Docs
HTML
def get_latest_version(opts)
- get_latest_github_release('jsdoc', 'jsdoc', opts)
+ get_npm_version('jsdoc', opts)
end
end
end
diff --git a/lib/docs/scrapers/julia.rb b/lib/docs/scrapers/julia.rb
index 8d5b4a3e..f7a03e53 100644
--- a/lib/docs/scrapers/julia.rb
+++ b/lib/docs/scrapers/julia.rb
@@ -11,6 +11,17 @@ module Docs
Licensed under the MIT License.
HTML
+ version '1.7' do
+ self.release = '1.7.0'
+ self.base_url = "https://docs.julialang.org/en/v#{release}/"
+ self.type = 'julia'
+
+ html_filters.push 'julia/entries', 'julia/clean_html'
+
+ options[:container] = '.docs-main'
+ options[:only_patterns] = [/\Amanual\//, /\Abase\//, /\Astdlib\//]
+ end
+
version '1.6' do
self.release = '1.6.0'
self.base_url = "https://docs.julialang.org/en/v#{release}/"
diff --git a/lib/docs/scrapers/kotlin.rb b/lib/docs/scrapers/kotlin.rb
index 91ca19ff..38882986 100644
--- a/lib/docs/scrapers/kotlin.rb
+++ b/lib/docs/scrapers/kotlin.rb
@@ -1,7 +1,6 @@
module Docs
class Kotlin < UrlScraper
self.type = 'kotlin'
- self.release = '1.4.10'
self.base_url = 'https://kotlinlang.org/'
self.root_path = 'api/latest/jvm/stdlib/index.html'
self.links = {
@@ -11,9 +10,8 @@ module Docs
html_filters.push 'kotlin/entries', 'kotlin/clean_html'
- options[:container] = '.global-content'
-
- options[:only_patterns] = [/\Adocs\/tutorials\//, /\Adocs\/reference\//, /\Aapi\/latest\/jvm\/stdlib\//]
+ options[:container] = 'article'
+ options[:only_patterns] = [/\Adocs\//, /\Aapi\/latest\/jvm\/stdlib\//]
options[:skip_patterns] = [/stdlib\/org\./]
options[:skip] = %w(
api/latest/jvm/stdlib/alltypes/index.html
@@ -22,23 +20,39 @@ module Docs
docs/events.html
docs/resources.html
docs/reference/grammar.html)
- options[:replace_paths] = {
- 'api/latest/jvm/stdlib/' => 'api/latest/jvm/stdlib/index.html',
- 'docs/reference/coroutines.html' => 'docs/reference/coroutines-overview.html',
- 'api/latest/jvm/stdlib/kotlin/fold.html' => 'api/latest/jvm/stdlib/kotlin.collections/fold.html',
- 'api/latest/jvm/stdlib/kotlin/get-or-else.html' => 'api/latest/jvm/stdlib/kotlin.collections/get-or-else.html',
- 'api/latest/jvm/stdlib/kotlin/map.html' => 'api/latest/jvm/stdlib/kotlin.collections/map.html',
- 'docs/tutorials/native/targeting-multiple-platforms.html' => 'docs/tutorials/native/basic-kotlin-native-app.html',
- 'api/latest/jvm/stdlib/kotlin/-throwable/print-stack-trace.html' => 'api/latest/jvm/stdlib/kotlin/print-stack-trace.html',
- }
+
+ options[:fix_urls] = ->(url) do
+ url.sub! %r{/docs/reference/}, '/docs/'
+ url
+ end
options[:attribution] = <<-HTML
- © 2010–2020 JetBrains s.r.o. and Kotlin Programming Language contributors
+ © 2010–2022 JetBrains s.r.o. and Kotlin Programming Language contributors
Licensed under the Apache License, Version 2.0.
HTML
+ version '1.6' do
+ self.release = '1.6.20'
+ end
+
+ version '1.4' do
+ self.release = '1.4.10'
+ end
+
def get_latest_version(opts)
get_latest_github_release('JetBrains', 'kotlin', opts)
end
+
+ private
+
+ def process_response?(response)
+ return false unless super
+ response.body !~ /http-equiv="refresh"/i
+ end
+
+ def parse(response)
+ response.body.gsub! %r{]*>([\W\w]+?)
}, '\2
'
+ super
+ end
end
end
diff --git a/lib/docs/scrapers/kubectl.rb b/lib/docs/scrapers/kubectl.rb
new file mode 100644
index 00000000..0f888ddf
--- /dev/null
+++ b/lib/docs/scrapers/kubectl.rb
@@ -0,0 +1,38 @@
+module Docs
+ class Kubectl < UrlScraper
+ self.name = 'Kubectl'
+ self.type = 'kubectl'
+ self.root_path = ''
+ self.links = {
+ home: 'https://kubernetes.io/docs/reference/kubectl/',
+ code: 'https://github.com/kubernetes/kubernetes'
+ }
+ self.release = "1.23"
+ self.base_url = "https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands"
+
+ html_filters.push 'kubectl/entries', 'kubectl/clean_html'
+
+ options[:container] = '#page-content-wrapper'
+
+ options[:attribution] = <<-HTML
+ © 2022 The Kubernetes Authors | Documentation Distributed under CC BY 4.0
+ Copyright © 2022 The Linux Foundation ®. All rights reserved.
+ HTML
+
+ # latest version has a special URL that does not include the version identifier
+ version do
+ self.release = "1.23"
+ self.base_url = "https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands"
+ end
+
+ version '1.20' do
+ self.release = "#{version}"
+ self.base_url = "https://v#{version.sub('.', '-')}.docs.kubernetes.io/docs/reference/generated/kubectl/kubectl-commands"
+ end
+
+ def get_latest_version(opts)
+ get_latest_github_release('kubernetes', 'kubernetes', opts)
+ end
+
+ end
+end
diff --git a/lib/docs/scrapers/kubernetes.rb b/lib/docs/scrapers/kubernetes.rb
new file mode 100644
index 00000000..143c16ae
--- /dev/null
+++ b/lib/docs/scrapers/kubernetes.rb
@@ -0,0 +1,37 @@
+module Docs
+ class Kubernetes < UrlScraper
+ self.name = 'Kubernetes'
+ self.type = 'kubernetes'
+ self.root_path = '/'
+ self.links = {
+ home: 'https://kubernetes.io/',
+ code: 'https://github.com/kubernetes/kubernetes'
+ }
+
+ # https://kubernetes.io/docs/reference/kubernetes-api/
+ html_filters.push 'kubernetes/entries', 'kubernetes/clean_html'
+
+ options[:container] = '.td-content'
+
+ options[:attribution] = <<-HTML
+ © 2022 The Kubernetes Authors | Documentation Distributed under CC BY 4.0
+ Copyright © 2022 The Linux Foundation ®. All rights reserved.
+ HTML
+
+ # latest version has a special URL that does not include the version identifier
+ version do
+ self.release = "1.23"
+ self.base_url = "https://kubernetes.io/docs/reference/kubernetes-api/"
+ end
+
+ version '1.20' do
+ self.release = "#{version}"
+ self.base_url = "https://v#{version.sub('.', '-')}.docs.kubernetes.io/docs/reference/kubernetes-api/"
+ end
+
+ def get_latest_version(opts)
+ get_latest_github_release('kubernetes', 'kubernetes', opts)
+ end
+
+ end
+end
diff --git a/lib/docs/scrapers/latex.rb b/lib/docs/scrapers/latex.rb
index e603b72a..816f1d64 100644
--- a/lib/docs/scrapers/latex.rb
+++ b/lib/docs/scrapers/latex.rb
@@ -1,8 +1,10 @@
+# coding: utf-8
module Docs
class Latex < UrlScraper
self.name = 'LaTeX'
self.slug = 'latex'
self.type = 'simple'
+ self.release = 'April 2021'
self.links = {
home: 'https://ctan.org/pkg/latex2e-help-texinfo/'
}
@@ -18,5 +20,12 @@ module Docs
Public Domain Software
HTML
+ def get_latest_version(opts)
+ body = fetch('https://latexref.xyz/', opts)
+ body = body.scan(/\(\w+\s\d+\)/)[0]
+ body.sub!('(', '')
+ body.sub!(')', '')
+ end
+
end
end
diff --git a/lib/docs/scrapers/leaflet.rb b/lib/docs/scrapers/leaflet.rb
index 69abef25..52bae4ff 100644
--- a/lib/docs/scrapers/leaflet.rb
+++ b/lib/docs/scrapers/leaflet.rb
@@ -14,11 +14,16 @@ module Docs
options[:skip_links] = true
options[:attribution] = <<-HTML
- © 2010–2019 Vladimir Agafonkin
+ © 2010–2022 Vladimir Agafonkin
© 2010–2011, CloudMade
Maps © OpenStreetMap contributors.
HTML
+ version '1.8' do
+ self.release = '1.8.0'
+ self.base_url = "https://leafletjs.com/SlavaUkraini/reference.html"
+ end
+
version '1.7' do
self.release = '1.7.1'
self.base_url = "https://leafletjs.com/reference-#{release}.html"
@@ -31,7 +36,7 @@ module Docs
version '1.5' do
self.release = '1.5.1'
- self.base_url = "https://leafletjs.com/reference-1.5.0.html"
+ self.base_url = "https://leafletjs.com/reference-#{release}.html"
end
version '1.4' do
@@ -60,9 +65,9 @@ module Docs
end
def get_latest_version(opts)
- doc = fetch_doc('https://leafletjs.com/index.html', opts)
- link = doc.css('ul > li > a').to_a.select {|node| node.content == 'Docs'}.first
- link['href'].scan(/reference-([0-9.]+)\.html/)[0][0]
+ doc = fetch_doc('https://leafletjs.com/reference-versions.html', opts)
+ link = doc.at_css('.container > ul > li:last-child > a').content
+ link.sub(/[a-zA-Z\s]*/, '')
end
end
end
diff --git a/lib/docs/scrapers/mariadb.rb b/lib/docs/scrapers/mariadb.rb
index fedc1e5e..cf2c69c3 100644
--- a/lib/docs/scrapers/mariadb.rb
+++ b/lib/docs/scrapers/mariadb.rb
@@ -2,9 +2,9 @@ module Docs
class Mariadb < UrlScraper
self.name = 'MariaDB'
self.type = 'mariadb'
- self.release = '10.4.8'
+ self.release = '10.6.4'
self.base_url = 'https://mariadb.com/kb/en/'
- self.root_path = 'library/documentation/'
+ self.root_path = 'documentation/'
self.links = {
home: 'https://mariadb.com/',
code: 'https://github.com/MariaDB/server'
@@ -24,13 +24,14 @@ module Docs
]
options[:attribution] = <<-HTML
- © 2019 MariaDB
+ © 2021 MariaDB
Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License.
HTML
def get_latest_version(opts)
doc = fetch_doc('https://mariadb.com/downloads/', opts)
- doc.at_css('[data-version-id="mariadb_server-versions"] option').content.split('-')[0]
+ doc.at_css('#version-select-community_server > option').content.split('-')[0]
end
+
end
end
diff --git a/lib/docs/scrapers/matplotlib.rb b/lib/docs/scrapers/matplotlib.rb
index a76dcc61..a2add6bf 100644
--- a/lib/docs/scrapers/matplotlib.rb
+++ b/lib/docs/scrapers/matplotlib.rb
@@ -12,7 +12,7 @@ module Docs
html_filters.push 'matplotlib/entries', 'sphinx/clean_html'
- options[:container] = '.body'
+ options[:container] = '.body, section'
options[:skip] = %w(api_changes.html tutorial.html faq.html)
options[:attribution] = <<-HTML
@@ -20,8 +20,17 @@ module Docs
Licensed under the Matplotlib License Agreement.
HTML
+ version '3.5' do
+ self.release = '3.5.1'
+ self.base_urls = [
+ "https://matplotlib.org/#{release}/api/",
+ "https://matplotlib.org/#{release}/mpl_toolkits/mplot3d/",
+ "https://matplotlib.org/#{release}/mpl_toolkits/axes_grid/api/"
+ ]
+ end
+
version '3.4' do
- self.release = '3.4.1'
+ self.release = '3.4.3'
self.base_urls = [
"https://matplotlib.org/#{release}/api/",
"https://matplotlib.org/#{release}/mpl_toolkits/mplot3d/",
diff --git a/lib/docs/scrapers/mdn/css.rb b/lib/docs/scrapers/mdn/css.rb
index 98a126ac..8405997a 100644
--- a/lib/docs/scrapers/mdn/css.rb
+++ b/lib/docs/scrapers/mdn/css.rb
@@ -1,5 +1,6 @@
module Docs
class Css < Mdn
+ # release = '2022-04-27'
self.name = 'CSS'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/CSS'
self.root_path = '/Reference'
diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb
index b75af630..a28a8120 100644
--- a/lib/docs/scrapers/mdn/dom.rb
+++ b/lib/docs/scrapers/mdn/dom.rb
@@ -1,12 +1,14 @@
module Docs
class Dom < Mdn
- self.name = 'DOM'
+ # release = '2022-04-27'
+ self.name = 'Web APIs'
+ self.slug = 'dom'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'
html_filters.push 'dom/clean_html', 'dom/entries'
- options[:root_title] = 'DOM'
+ options[:root_title] = 'Web APIs'
end
end
diff --git a/lib/docs/scrapers/mdn/html.rb b/lib/docs/scrapers/mdn/html.rb
index 1f108711..9a72cd3c 100644
--- a/lib/docs/scrapers/mdn/html.rb
+++ b/lib/docs/scrapers/mdn/html.rb
@@ -2,6 +2,7 @@ module Docs
class Html < Mdn
prepend FixInternalUrlsBehavior
+ # release = '2022-04-27'
self.name = 'HTML'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/HTML'
diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb
index 8238d3b4..d8b9f969 100644
--- a/lib/docs/scrapers/mdn/javascript.rb
+++ b/lib/docs/scrapers/mdn/javascript.rb
@@ -3,6 +3,7 @@ module Docs
prepend FixInternalUrlsBehavior
prepend FixRedirectionsBehavior
+ # release = '2022-04-28'
self.name = 'JavaScript'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference'
diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb
index 2b31668a..977079a8 100644
--- a/lib/docs/scrapers/mdn/mdn.rb
+++ b/lib/docs/scrapers/mdn/mdn.rb
@@ -13,7 +13,7 @@ module Docs
}
options[:attribution] = <<-HTML
- © 2005–2021 MDN contributors.
+ © 2005–2022 MDN contributors.
Licensed under the Creative Commons Attribution-ShareAlike License v2.5 or later.
HTML
diff --git a/lib/docs/scrapers/mdn/svg.rb b/lib/docs/scrapers/mdn/svg.rb
index 4a9de544..b5a22b06 100644
--- a/lib/docs/scrapers/mdn/svg.rb
+++ b/lib/docs/scrapers/mdn/svg.rb
@@ -3,6 +3,7 @@ module Docs
prepend FixInternalUrlsBehavior
prepend FixRedirectionsBehavior
+ # release = '2022-04-20'
self.name = 'SVG'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/SVG'
diff --git a/lib/docs/scrapers/mdn/web_extensions.rb b/lib/docs/scrapers/mdn/web_extensions.rb
index 1a5f528a..c0c49185 100644
--- a/lib/docs/scrapers/mdn/web_extensions.rb
+++ b/lib/docs/scrapers/mdn/web_extensions.rb
@@ -1,5 +1,6 @@
module Docs
class WebExtensions < Mdn
+ # release = '2022-04-28'
self.name = 'Web Extensions'
self.slug = 'web_extensions'
self.links = {
diff --git a/lib/docs/scrapers/mdn/xslt_xpath.rb b/lib/docs/scrapers/mdn/xslt_xpath.rb
index 286c48f4..26a298b6 100644
--- a/lib/docs/scrapers/mdn/xslt_xpath.rb
+++ b/lib/docs/scrapers/mdn/xslt_xpath.rb
@@ -1,5 +1,6 @@
module Docs
class XsltXpath < Mdn
+ # release = '2021-10-22'
self.name = 'XSLT & XPath'
self.slug = 'xslt_xpath'
self.base_url = 'https://developer.mozilla.org/en-US/docs/Web'
diff --git a/lib/docs/scrapers/mkdocs/django_rest_framework.rb b/lib/docs/scrapers/mkdocs/django_rest_framework.rb
index db58eb8c..54ea2694 100644
--- a/lib/docs/scrapers/mkdocs/django_rest_framework.rb
+++ b/lib/docs/scrapers/mkdocs/django_rest_framework.rb
@@ -1,7 +1,7 @@
module Docs
class DjangoRestFramework < Mkdocs
self.name = 'Django REST Framework'
- self.release = '3.9.3'
+ self.release = '3.12.4'
self.slug = 'django_rest_framework'
self.base_url = 'https://www.django-rest-framework.org/'
self.root_path = 'index.html'
diff --git a/lib/docs/scrapers/mocha.rb b/lib/docs/scrapers/mocha.rb
index db27a076..99ce4d3e 100644
--- a/lib/docs/scrapers/mocha.rb
+++ b/lib/docs/scrapers/mocha.rb
@@ -1,7 +1,7 @@
module Docs
class Mocha < UrlScraper
self.type = 'simple'
- self.release = '8.3.2'
+ self.release = '9.0.2'
self.base_url = 'https://mochajs.org/'
self.links = {
home: 'https://mochajs.org/',
diff --git a/lib/docs/scrapers/nginx.rb b/lib/docs/scrapers/nginx.rb
index aa7c2b7a..b28c0e7a 100644
--- a/lib/docs/scrapers/nginx.rb
+++ b/lib/docs/scrapers/nginx.rb
@@ -2,7 +2,7 @@ module Docs
class Nginx < UrlScraper
self.name = 'nginx'
self.type = 'nginx'
- self.release = '1.20.0'
+ self.release = '1.22.0'
self.base_url = 'https://nginx.org/en/docs/'
self.links = {
home: 'https://nginx.org/',
@@ -23,7 +23,7 @@ module Docs
# http://nginx.org/LICENSE
options[:attribution] = <<-HTML
© 2002-2021 Igor Sysoev
- © 2011-2021 Nginx, Inc.
+ © 2011-2022 Nginx, Inc.
Licensed under the BSD License.
HTML
diff --git a/lib/docs/scrapers/nim.rb b/lib/docs/scrapers/nim.rb
index 6f17c4fb..5a9d4f00 100644
--- a/lib/docs/scrapers/nim.rb
+++ b/lib/docs/scrapers/nim.rb
@@ -1,7 +1,7 @@
module Docs
class Nim < UrlScraper
self.type = 'simple'
- self.release = '1.4.2'
+ self.release = '1.4.8'
self.base_url = 'https://nim-lang.org/docs/'
self.root_path = 'overview.html'
self.links = {
@@ -14,7 +14,7 @@ module Docs
options[:skip] = %w(theindex.html docgen.html tut1.html tut2.html tut3.html tools.html)
options[:attribution] = <<-HTML
- © 2006–2020 Andreas Rumpf
+ © 2006–2021 Andreas Rumpf
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/nix.rb b/lib/docs/scrapers/nix.rb
new file mode 100644
index 00000000..ae549818
--- /dev/null
+++ b/lib/docs/scrapers/nix.rb
@@ -0,0 +1,31 @@
+module Docs
+ class Nix < UrlScraper
+ self.type = 'simple'
+ self.release = '2.8.0'
+ self.base_url = 'https://nixos.org/manual/'
+ self.root_path = 'nix/stable/expressions/builtins.html'
+ self.initial_paths = %w(
+ nix/stable/expressions/builtins.html
+ nixpkgs/stable/index.html)
+ self.links = {
+ home: 'https://nixos.org/',
+ code: 'https://github.com/NixOS/nix'
+ }
+
+ html_filters.push 'nix/clean_html', 'nix/entries'
+
+ options[:skip_links] = true
+
+ options[:attribution] = <<-HTML
+ © 2022 NixOS Contributors
+ Licensed under the LGPL License.
+ HTML
+
+ def get_latest_version(opts)
+ doc = fetch_doc('https://nixos.org/manual/nix/stable/', opts)
+ json = JSON.parse(doc.at_css('body')['data-nix-channels'])
+ channel = json.find { |c| c['channel'] == 'stable' }
+ channel['version']
+ end
+ end
+end
diff --git a/lib/docs/scrapers/node.rb b/lib/docs/scrapers/node.rb
index 815d61f7..8d5db646 100644
--- a/lib/docs/scrapers/node.rb
+++ b/lib/docs/scrapers/node.rb
@@ -24,10 +24,15 @@ module Docs
HTML
version do
- self.release = '16.2.0'
+ self.release = '18.0.0'
self.base_url = 'https://nodejs.org/api/'
end
+ version '16 LTS' do
+ self.release = '16.13.2'
+ self.base_url = 'https://nodejs.org/dist/latest-v16.x/docs/api/'
+ end
+
version '14 LTS' do
self.release = '14.17.0'
self.base_url = 'https://nodejs.org/dist/latest-v14.x/docs/api/'
diff --git a/lib/docs/scrapers/nokogiri2.rb b/lib/docs/scrapers/nokogiri2.rb
index 0bb975c0..cef6e565 100644
--- a/lib/docs/scrapers/nokogiri2.rb
+++ b/lib/docs/scrapers/nokogiri2.rb
@@ -7,7 +7,8 @@ module Docs
self.name = 'Nokogiri'
self.slug = 'nokogiri'
- self.release = '1.10.4'
+ self.release = '1.13.0'
+ self.base_url = "https://nokogiri.org/rdoc/"
html_filters.replace 'rdoc/entries', 'nokogiri2/entries'
@@ -15,7 +16,7 @@ module Docs
options[:only_patterns] = [/\ANokogiri/, /\AXSD/]
options[:attribution] = <<-HTML
- © 2008–2018 Aaron Patterson, Mike Dalessio, Charles Nutter, Sergio Arbeo,
+ © 2008–2021 Aaron Patterson, Mike Dalessio, Charles Nutter, Sergio Arbeo,
Patrick Mahoney, Yoko Harada, Akinori MUSHA, John Shahid, Lars Kanis
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/npm.rb b/lib/docs/scrapers/npm.rb
index 7816c365..ce2be86b 100644
--- a/lib/docs/scrapers/npm.rb
+++ b/lib/docs/scrapers/npm.rb
@@ -2,7 +2,7 @@ module Docs
class Npm < UrlScraper
self.name = 'npm'
self.type = 'npm'
- self.release = '7.10.0'
+ self.release = '8.3.0'
self.base_url = 'https://docs.npmjs.com/'
self.force_gzip = true
self.links = {
@@ -18,16 +18,20 @@ module Docs
'all',
'misc/index',
'cli',
- 'orgs/',
+ 'organizations/',
+ 'orgs',
'removing-members-from-your-org',
'adding-members-to-your-org',
- 'downloading-and-installing-packages'
+ 'downloading-and-installing-packages',
]
options[:skip_patterns] = [
/\Aenterprise/,
/\Acompany/,
- /\Apolicies/
+ /\Apolicies/,
+ /cli\/v6/,
+ /cli\/v7/,
+ /\/\Z/ # avoid pages with a trailing slash, those pages mess up the entries
]
options[:attribution] = <<-HTML
@@ -39,36 +43,38 @@ module Docs
# fix duplicates
options[:fix_urls] = -> (url) do
url.sub!('private-modules/intro', 'creating-and-publishing-private-packages')
- url.sub!('cli/audit', 'cli/v6/commands/npm-audit')
- url.sub!('cli/uninstall', 'cli/v6/commands/npm-uninstall')
- url.sub!('cli/npm', 'cli/v6/commands/npm')
- url.sub!('cli-documentation', 'cli/v6')
- url.sub!('misc/registry', 'cli/v6/using-npm/registry')
- url.sub!('cli/adduser', 'cli/v6/commands/npm-adduser')
- url.sub!('cli/profile', 'cli/v6/commands/npm-profile')
- url.sub!('cli/token', 'cli/v6/commands/npm-token')
- url.sub!('cli/publish', 'cli/v6/commands/npm-publish')
- url.sub!('cli/unpublish', 'cli/v6/commands/npm-unpublish')
- url.sub!('cli/deprecate', 'cli/v6/commands/npm-deprecate')
- url.sub!('cli/access', 'cli/v6/commands/npm-access')
- url.sub!('misc/config', 'cli/v6/using-npm/config')
- url.sub!('misc/developers', 'cli/v6/using-npm/developers')
url.sub!('managing-team-access-to-packages', 'managing-team-access-to-organization-packages')
url.sub!('accepting-or-rejecting-an-org-invitation', 'accepting-or-rejecting-an-organization-invitation')
url.sub!('org-roles-and-permissions', 'organization-roles-and-permissions')
url.sub!('upgrading-to-a-paid-org-plan', 'upgrading-to-a-paid-organization-plan')
- url.sub!('files/package.json', 'cli/v6/configuring-npm/package-json')
url.sub!('managing-team-access-to-org-packages', 'managing-team-access-to-organization-packages')
url.sub!('about-package-json-and-package-lock-json-files', 'creating-a-package-json-file')
- url.sub!('cli/team', 'cli/v6/commands/npm-team')
- url.sub!('cli/version', 'cli/v6/commands/npm-version')
- url.sub!('creating-a-packge-json-file', 'cli/v6/configuring-npm/package-json')
- url.sub!('cli/outdated', 'cli/v6/commands/npm-outdated')
- url.sub!('cli/owner', 'cli/v6/commands/npm-owner')
- url.sub!('cli/install', 'cli/v6/commands/npm-install')
- url.sub!('cli/update', 'cli/v6/commands/npm-update')
- url.sub!('cli/config', 'cli/v6/commands/npm-config')
- url.sub!('cli/dist-tag', 'cli/v6/commands/npm-dist-tag')
+ url.sub!('cli/npm', 'cli/v8/commands/npm/')
+ url.sub!('cli/config', 'cli/v8/commands/npm-config/')
+ url.sub!('misc/registry', 'cli/v8/using-npm/registry/')
+ url.sub!('cli-documentation', 'cli/v8')
+ url.sub!('cli-documentation/files/npmrc', 'cli/v8/configuring-npm/npmrc/')
+ url.sub!('configuring-your-registry-settings-as-an-npm-enterprise-user', 'enterprise')
+ url.sub!('cli/publish', 'cli/v8/commands/npm-publish/')
+ url.sub!('cli/deprecate', 'cli/v8/commands/npm-deprecate/')
+ url.sub!('cli/access', 'cli/v8/commands/npm-access/')
+ url.sub!('cli/adduser', 'cli/v8/commands/npm-adduser/')
+ url.sub!('misc/config', 'cli/v8/using-npm/config/')
+ url.sub!('cli/token', 'cli/v8/commands/npm-token/')
+ url.sub!('cli/unpublish', 'cli/v8/commands/npm-unpublish/')
+ url.sub!('files/package.json', 'cli/v8/configuring-npm/package-json/')
+ url.sub!('cli/profile', 'cli/v8/commands/npm-profile/')
+ url.sub!('creating-a-packge-json-file', 'cli/v8/configuring-npm/package-json/')
+ url.sub!('cli/dist-tag', 'cli/v8/commands/npm-dist-tag/')
+ url.sub!('cli/team', 'cli/v8/commands/npm-team/')
+ url.sub!('cli/version', 'cli/v8/commands/npm-version/')
+ url.sub!('cli/owner', 'cli/v8/commands/npm-owner/')
+ url.sub!('cli/install', '/cli/v8/commands/npm-install/')
+ url.sub!('cli/audit', 'cli/v8/commands/npm-audit/')
+ url.sub!('cli/update', 'cli/v8/commands/npm-update/')
+ url.sub!('cli/outdated', 'cli/v8/commands/npm-outdated/')
+ url.sub!('cli/uninstall', 'cli/v8/commands/npm-uninstall/')
+ url.sub!('misc/developers', 'cli/v8/using-npm/developers//')
url
end
diff --git a/lib/docs/scrapers/numpy.rb b/lib/docs/scrapers/numpy.rb
index 50fc304e..22fba843 100644
--- a/lib/docs/scrapers/numpy.rb
+++ b/lib/docs/scrapers/numpy.rb
@@ -26,8 +26,20 @@ module Docs
Licensed under the 3-clause BSD License.
HTML
+ version '1.22' do
+ self.release = '1.22.0'
+ self.base_url = "https://numpy.org/doc/#{self.version}/"
+ options[:container] = nil
+ end
+
+ version '1.21' do
+ self.release = '1.21.5'
+ self.base_url = "https://numpy.org/doc/#{self.version}/"
+ options[:container] = nil
+ end
+
version '1.20' do
- self.release = '1.20.1'
+ self.release = '1.20.3'
self.base_url = "https://numpy.org/doc/#{self.version}/"
options[:container] = nil
end
diff --git a/lib/docs/scrapers/ocaml.rb b/lib/docs/scrapers/ocaml.rb
index 9f5cab47..72de98f8 100644
--- a/lib/docs/scrapers/ocaml.rb
+++ b/lib/docs/scrapers/ocaml.rb
@@ -3,7 +3,7 @@ module Docs
self.name = 'OCaml'
self.type = 'ocaml'
self.root_path = 'index.html'
- self.release = '4.12'
+ self.release = '4.14'
self.base_url = "https://www.ocaml.org/releases/#{self.release}/htmlman/"
self.links = {
home: 'https://ocaml.org/',
@@ -23,7 +23,7 @@ module Docs
]
options[:attribution] = <<-HTML
- © 1995-2021 INRIA.
+ © 1995-2022 INRIA.
HTML
def get_latest_version(opts)
diff --git a/lib/docs/scrapers/octave.rb b/lib/docs/scrapers/octave.rb
index 775afe19..3a7c38d3 100644
--- a/lib/docs/scrapers/octave.rb
+++ b/lib/docs/scrapers/octave.rb
@@ -23,14 +23,19 @@ module Docs
options[:root_title] = 'GNU Octave'
options[:attribution] = <<-HTML
- © 1996–2020 John W. Eaton
+ © 1996–2022 The Octave Project Developers
Permission is granted to make and distribute verbatim copies of this manual provided the copyright notice and this permission notice are preserved on all copies.
Permission is granted to copy and distribute modified versions of this manual under the conditions for verbatim copying, provided that the entire resulting derived work is distributed under the terms of a permission notice identical to this one.
Permission is granted to copy and distribute translations of this manual into another language, under the above conditions for modified versions.
HTML
+ version '7' do
+ self.release = '7.1.0'
+ self.base_url = "https://octave.org/doc/v#{self.release}/"
+ end
+
version '6' do
- self.release = '6.2.0'
+ self.release = '6.4.0'
self.base_url = "https://octave.org/doc/v#{self.release}/"
end
diff --git a/lib/docs/scrapers/openjdk.rb b/lib/docs/scrapers/openjdk.rb
index bb453432..6a8fdb96 100644
--- a/lib/docs/scrapers/openjdk.rb
+++ b/lib/docs/scrapers/openjdk.rb
@@ -28,7 +28,7 @@ module Docs
]
options[:attribution] = <<-HTML
- © 1993, 2020, Oracle and/or its affiliates. All rights reserved.
+ © 1993, 2022, Oracle and/or its affiliates. All rights reserved.
Documentation extracted from Debian's OpenJDK Development Kit package.
Licensed under the GNU General Public License, version 2, with the Classpath Exception.
Various third party code in OpenJDK is licensed under different licenses (see Debian package).
@@ -37,10 +37,20 @@ module Docs
NEWFILTERS = ['openjdk/entries_new', 'openjdk/clean_html_new']
- version '15' do
- self.release = '15.0.1'
+ version '18' do
+ self.release = '18'
self.root_path = 'index.html'
- self.base_url = 'https://docs.oracle.com/en/java/javase/15/docs/api/'
+ self.base_url = 'https://docs.oracle.com/en/java/javase/18/docs/api/'
+
+ html_filters.push NEWFILTERS
+
+ options[:container] = 'main'
+ end
+
+ version '17' do
+ self.release = '17'
+ self.root_path = 'index.html'
+ self.base_url = 'https://docs.oracle.com/en/java/javase/17/docs/api/'
html_filters.push NEWFILTERS
@@ -50,7 +60,7 @@ module Docs
OLDFILTERS = ['openjdk/entries', 'openjdk/clean_html']
version '11' do
- self.release = '11.0.10'
+ self.release = '11.0.11'
self.root_path = 'index.html'
self.base_url = 'https://docs.oracle.com/en/java/javase/11/docs/api/'
@@ -132,23 +142,11 @@ module Docs
end
def get_latest_version(opts)
- latest_version = 8
- current_attempt = latest_version
- attempts = 0
-
- while attempts < 3
- current_attempt += 1
-
- doc = fetch_doc("https://packages.debian.org/sid/openjdk-#{current_attempt}-doc", opts)
- if doc.at_css('.perror').nil?
- latest_version = current_attempt
- attempts = 0
- else
- attempts += 1
- end
- end
-
- latest_version
+ doc = fetch_doc("https://jdk.java.net/archive/", opts)
+ version = doc.at_css('#downloads > table > tr > th').content
+ version.gsub!(/\(.*\)/, '')
+ version.gsub!(/[a-zA-z]/, '')
+ version
end
end
end
diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb
index b1eccb71..f95bf575 100644
--- a/lib/docs/scrapers/pandas.rb
+++ b/lib/docs/scrapers/pandas.rb
@@ -1,5 +1,5 @@
module Docs
- class Pandas < UrlScraper
+ class Pandas < FileScraper
self.name = 'pandas'
self.type = 'sphinx'
self.root_path = 'index.html'
@@ -8,22 +8,22 @@ module Docs
code: 'https://github.com/pydata/pandas'
}
- html_filters.push 'pandas/clean_html', 'pandas/entries'
-
options[:skip] = %w(internals.html release.html contributing.html whatsnew.html)
options[:skip_patterns] = [/whatsnew\//]
options[:attribution] = <<-HTML
- © 2008–2020, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
+ © 2008–2022, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team
Licensed under the 3-clause BSD License.
HTML
version '1' do
- self.release = '1.2.0'
+ self.release = '1.4.0'
self.base_url = "https://pandas.pydata.org/pandas-docs/version/#{self.release}/"
html_filters.push 'pandas/clean_html', 'pandas/entries'
+ options[:container] = 'main section'
+
options[:skip_patterns] = [
/development/,
/getting_started/,
diff --git a/lib/docs/scrapers/perl.rb b/lib/docs/scrapers/perl.rb
index 90844c47..740b3ba4 100644
--- a/lib/docs/scrapers/perl.rb
+++ b/lib/docs/scrapers/perl.rb
@@ -2,7 +2,6 @@ module Docs
class Perl < UrlScraper
self.name = 'Perl'
self.type = 'perl'
-# self.root_path = 'index.html'
self.initial_paths = ['modules.html', 'perlutil.html', 'perl.html']
self.links = {
home: 'https://www.perl.org/'
@@ -20,44 +19,54 @@ module Docs
options[:skip_patterns] = [/\Afunctions/, /\Avariables/, /\.pdf/, /delta/]
options[:attribution] = <<-HTML
- © 1993–2020 Larry Wall and others
+ © 1993–2021 Larry Wall and others
Licensed under the GNU General Public License version 1 or later, or the Artistic License.
The Perl logo is a trademark of the Perl Foundation.
HTML
+ version '5.36' do
+ self.release = '5.36.0'
+ self.base_url = "https://perldoc.perl.org/#{release}/"
+ end
+
+ version '5.34' do
+ self.release = '5.34.0'
+ self.base_url = "https://perldoc.perl.org/#{release}/"
+ end
+
version '5.32' do
self.release = '5.32.0'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.30' do
self.release = '5.30.3'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.28' do
self.release = '5.28.3'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.26' do
self.release = '5.26.3'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.24' do
self.release = '5.24.4'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.22' do
self.release = '5.22.4'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
version '5.20' do
self.release = '5.20.3'
- self.base_url = "https://perldoc.perl.org/#{self.release}/"
+ self.base_url = "https://perldoc.perl.org/#{release}/"
end
def get_latest_version(opts)
diff --git a/lib/docs/scrapers/php.rb b/lib/docs/scrapers/php.rb
index 2db16d87..6facd0c9 100644
--- a/lib/docs/scrapers/php.rb
+++ b/lib/docs/scrapers/php.rb
@@ -5,7 +5,7 @@ module Docs
self.name = 'PHP'
self.type = 'php'
- self.release = '8.0'
+ self.release = '8.1.5'
self.base_url = 'https://www.php.net/manual/en/'
self.root_path = 'index.html'
self.initial_paths = %w(
@@ -62,7 +62,7 @@ module Docs
options[:skip_patterns] = [/mysqlnd/, /xdevapi/i]
options[:attribution] = <<-HTML
- © 1997–2020 The PHP Documentation Group
+ © 1997–2021 The PHP Documentation Group
Licensed under the Creative Commons Attribution License v3.0 or later.
HTML
diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb
new file mode 100644
index 00000000..f9ce6827
--- /dev/null
+++ b/lib/docs/scrapers/point_cloud_library.rb
@@ -0,0 +1,41 @@
+module Docs
+ class PointCloudLibrary < UrlScraper
+ self.name = 'PointCloudLibrary'
+ self.type = 'point_cloud_library'
+ self.slug = 'point_cloud_library'
+ self.base_url = 'https://pointclouds.org/documentation/'
+ self.root_path = 'modules.html'
+ # Add hierarchy.html to crawl all classes*.html that's not reachable from modules.html
+ self.initial_paths = [
+ "https://pointclouds.org/documentation/hierarchy.html"
+ ]
+ self.release = '1.12.1'
+
+ self.links = {
+ home: 'https://pointclouds.org/',
+ code: 'https://github.com/PointCloudLibrary/pcl'
+ }
+
+ html_filters.push 'point_cloud_library/entries', 'point_cloud_library/clean_html'
+
+ # Remove the `clean_text` because Doxygen are actually creating empty
+ # anchor such as to do anchor link.. and that anchor
+ # will be removed by clean_text
+ self.text_filters = FilterStack.new
+ text_filters.push 'images', 'inner_html', 'attribution'
+
+ def get_latest_version(opts)
+ get_latest_github_release('PointCloudLibrary', 'pcl', opts)[4..]
+ end
+
+ options[:attribution] = <<-HTML
+ © 2009–2012, Willow Garage, Inc.
+ © 2012–, Open Perception, Inc.
+ Licensed under the BSD License.
+ HTML
+
+ # Skip source code since it doesn't provide any useful docs
+ options[:skip_patterns] = [/_source/, /namespace/, /h\.html/, /structsvm/, /struct_/, /classopenni/, /class_/]
+
+ end
+end
diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb
index 1778b647..dfc371ee 100644
--- a/lib/docs/scrapers/postgresql.rb
+++ b/lib/docs/scrapers/postgresql.rb
@@ -51,12 +51,17 @@ module Docs
/\Aunsupported-features/ ]
options[:attribution] = <<-HTML
- © 1996–2021 The PostgreSQL Global Development Group
+ © 1996–2022 The PostgreSQL Global Development Group
Licensed under the PostgreSQL License.
HTML
+ version '14' do
+ self.release = '14.3'
+ self.base_url = "https://www.postgresql.org/docs/#{version}/"
+ end
+
version '13' do
- self.release = '13.2'
+ self.release = '13.4'
self.base_url = "https://www.postgresql.org/docs/#{version}/"
end
diff --git a/lib/docs/scrapers/prettier.rb b/lib/docs/scrapers/prettier.rb
new file mode 100644
index 00000000..e11f8d77
--- /dev/null
+++ b/lib/docs/scrapers/prettier.rb
@@ -0,0 +1,25 @@
+module Docs
+ class Prettier < UrlScraper
+ self.name = 'Prettier'
+ self.type = 'simple'
+ self.release = '2.6.0'
+ self.base_url = 'https://prettier.io/docs/en/'
+ self.links = {
+ home: 'https://prettier.io/',
+ code: 'https://github.com/prettier/prettier'
+ }
+
+ # Docusaurus like react_native
+ html_filters.push 'prettier/entries', 'prettier/clean_html'
+
+ options[:container] = '.docMainWrapper'
+
+ options[:attribution] = <<-HTML
+ © James Long and contributors
+ HTML
+
+ def get_latest_version(opts)
+ get_npm_version('prettier', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb
index 88c076dd..c01b96b6 100644
--- a/lib/docs/scrapers/python.rb
+++ b/lib/docs/scrapers/python.rb
@@ -1,28 +1,48 @@
module Docs
class Python < FileScraper
self.type = 'python'
- self.root_path = 'library/index.html'
+ self.root_path = 'index.html'
self.links = {
home: 'https://www.python.org/',
code: 'https://github.com/python/cpython'
}
- options[:only_patterns] = [/\Alibrary\//]
+ options[:only_patterns] = [
+ # /\Ac-api/,
+ /\Adistributing/,
+ # /\Adistutils/,
+ /\Aextending/,
+ /\Afaq/,
+ /\Ahowto/,
+ /\Aindex.html/,
+ # /\Ainstall/,
+ /\Ainstalling/,
+ /\Alibrary/,
+ /\Areference/,
+ /\Atutorial/,
+ /\Ausing/,
+ ]
options[:skip] = %w(
library/2to3.html
library/formatter.html
- library/index.html
library/intro.html
library/undoc.html
library/unittest.mock-examples.html
library/sunau.html)
options[:attribution] = <<-HTML
- © 2001–2021 Python Software Foundation
+ © 2001–2022 Python Software Foundation
Licensed under the PSF License.
HTML
+ version '3.10' do
+ self.release = '3.10.4'
+ self.base_url = "https://docs.python.org/#{self.version}/"
+
+ html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
+ end
+
version '3.9' do
self.release = '3.9.4'
self.base_url = 'https://docs.python.org/3.9/'
diff --git a/lib/docs/scrapers/qt.rb b/lib/docs/scrapers/qt.rb
index f69c68d2..640bbe14 100644
--- a/lib/docs/scrapers/qt.rb
+++ b/lib/docs/scrapers/qt.rb
@@ -103,6 +103,11 @@ module Docs
Licensed under the GNU Free Documentation License, Version 1.3.
HTML
+ version do
+ self.release = '6.2'
+ self.base_url = "https://doc.qt.io/qt-#{self.release}/"
+ end
+
version '6.1' do
self.release = '6.1'
self.base_url = "https://doc.qt.io/qt-#{self.release}/"
diff --git a/lib/docs/scrapers/r.rb b/lib/docs/scrapers/r.rb
new file mode 100644
index 00000000..28a97c2c
--- /dev/null
+++ b/lib/docs/scrapers/r.rb
@@ -0,0 +1,58 @@
+module Docs
+ class R < FileScraper
+ self.name = 'R'
+ self.slug = 'r'
+ self.type = 'simple'
+ self.release = '4.1.0'
+ self.links = {
+ home: 'https://www.r-project.org/',
+ code: 'https://svn.r-project.org/R/'
+ }
+
+ self.root_path = 'doc/html/packages.html'
+
+ html_filters.push 'r/entries', 'r/clean_html'
+
+ options[:skip_links] = false
+
+ options[:attribution] = <<-HTML
+ Copyright (©) 1999–2012 R Foundation for Statistical Computing.
+ Licensed under the GNU General Public License.
+ HTML
+
+ # Never want those
+ options[:skip_patterns] = [
+ /\/DESCRIPTION$/,
+ /\/NEWS(\.[^\/]*)?$/,
+ /\/doc\/index\.html$/,
+ /\/demo$/,
+ /\.pdf$/
+ ]
+
+ options[:replace_paths] = {
+ ## We want to fix links like so − but only if the targets don’t exist:
+ # 'library/MASS/html/cov.mve.html' => 'library/MASS/html/cov.rob.html'
+ ## Paths for target packages or keywords that do not have their own file
+ ## are generated in the entries filter from 00Index.html files
+ }
+
+ options[:skip] = %w(
+ doc/html/packages-head-utf8.html
+ doc/html/SearchOn.html
+ doc/html/Search.html
+ doc/html/UserManuals.html
+ doc/html/faq.html
+ doc/manual/R-FAQ.html
+ doc/manual/R-admin.html
+ doc/manual/R-exts.html
+ doc/manual/R-ints.html
+ doc/manual/R-lang.html
+ )
+
+ def get_latest_version(opts)
+ body = fetch('https://cran.r-project.org/src/base/NEWS', opts)
+ body.match(/CHANGES IN R ([\d.]+):/)[1]
+ end
+
+ end
+end
diff --git a/lib/docs/scrapers/rdoc/rails.rb b/lib/docs/scrapers/rdoc/rails.rb
index 907deed3..3f2e1f27 100644
--- a/lib/docs/scrapers/rdoc/rails.rb
+++ b/lib/docs/scrapers/rdoc/rails.rb
@@ -1,20 +1,12 @@
module Docs
class Rails < Rdoc
- # Instructions:
- # 1. Download a release at https://github.com/rails/rails/releases
- # 2. Open "railties/lib/rails/api/task.rb" and comment out any code related to sdoc ("configure_sdoc")
- # 3. Run "bundle install --without db && bundle exec rake rdoc" (in the Rails directory)
- # 4. Run "cd guides && bundle exec rake guides:generate:html"
- # 5. Copy the "guides/output" directory to "html/guides"
- # 6. Copy the "html" directory to "docs/rails~[version]"
-
include FixInternalUrlsBehavior
self.name = 'Ruby on Rails'
self.slug = 'rails'
self.initial_paths = %w(guides/index.html)
self.links = {
- home: 'http://rubyonrails.org/',
+ home: 'https://rubyonrails.org/',
code: 'https://github.com/rails/rails'
}
@@ -72,31 +64,35 @@ module Docs
options[:attribution] = ->(filter) do
if filter.slug.start_with?('guides')
<<-HTML
- © 2004–2020 David Heinemeier Hansson
+ © 2004–2021 David Heinemeier Hansson
Licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
HTML
else
<<-HTML
- © 2004–2020 David Heinemeier Hansson
+ © 2004–2021 David Heinemeier Hansson
Licensed under the MIT License.
HTML
end
end
+ version '7.0' do
+ self.release = '7.0.0'
+ end
+
version '6.1' do
- self.release = '6.1.1'
+ self.release = '6.1.4'
end
version '6.0' do
- self.release = '6.0.0'
+ self.release = '6.0.4'
end
version '5.2' do
- self.release = '5.2.2'
+ self.release = '5.2.6'
end
version '5.1' do
- self.release = '5.1.6'
+ self.release = '5.1.7'
end
version '5.0' do
@@ -113,7 +109,7 @@ module Docs
def get_latest_version(opts)
doc = fetch_doc('https://rubyonrails.org/', opts)
- doc.at_css('.version p a').content.scan(/\d\.\d*\.*\d*\.*\d*/)[0]
+ doc.at_css('.heading__button span').content.scan(/\d\.\d*\.*\d*\.*\d*/)[0]
end
end
end
diff --git a/lib/docs/scrapers/rdoc/ruby.rb b/lib/docs/scrapers/rdoc/ruby.rb
index ed7891b4..76ddd1fb 100644
--- a/lib/docs/scrapers/rdoc/ruby.rb
+++ b/lib/docs/scrapers/rdoc/ruby.rb
@@ -63,12 +63,16 @@ module Docs
/\AXMP/]
options[:attribution] = <<-HTML
- Ruby Core © 1993–2020 Yukihiro Matsumoto
+ Ruby Core © 1993–2022 Yukihiro Matsumoto
Licensed under the Ruby License.
Ruby Standard Library © contributors
Licensed under their own licenses.
HTML
+ version '3.1' do
+ self.release = '3.1.1'
+ end
+
version '3' do
self.release = '3.0.0'
end
diff --git a/lib/docs/scrapers/react.rb b/lib/docs/scrapers/react.rb
index eeba46aa..c76f0bad 100644
--- a/lib/docs/scrapers/react.rb
+++ b/lib/docs/scrapers/react.rb
@@ -2,7 +2,7 @@ module Docs
class React < UrlScraper
self.name = 'React'
self.type = 'simple'
- self.release = '17.0.2'
+ self.release = '18.1.0'
self.base_url = 'https://reactjs.org/docs/'
self.root_path = 'hello-world.html'
self.links = {
diff --git a/lib/docs/scrapers/react_native.rb b/lib/docs/scrapers/react_native.rb
index d00e55c2..39249b4e 100644
--- a/lib/docs/scrapers/react_native.rb
+++ b/lib/docs/scrapers/react_native.rb
@@ -1,19 +1,19 @@
module Docs
class ReactNative < UrlScraper
+ self.name = 'React Native'
self.slug = 'react_native'
self.type = 'react_native'
- self.release = '0.56'
- self.base_url = 'https://facebook.github.io/react-native/docs/'
+ self.release = '0.66'
+ self.base_url = 'https://reactnative.dev/docs/'
self.root_path = 'getting-started.html'
self.links = {
- home: 'https://facebook.github.io/react-native/',
+ home: 'https://reactnative.dev/',
code: 'https://github.com/facebook/react-native'
}
html_filters.push 'react_native/entries', 'react_native/clean_html'
- options[:container] = '.docMainWrapper'
- options[:skip_patterns] = [/\Asample\-/]
+ options[:skip_patterns] = [/\Asample\-/, /\A0\./, /\Anext\b/]
options[:skip] = %w(
videos.html
transforms.html
@@ -26,8 +26,9 @@ module Docs
url
}
+ # https://github.com/facebook/react-native-website/blob/main/LICENSE-docs
options[:attribution] = <<-HTML
- © 2015–2018 Facebook Inc.
+ © 2022 Facebook Inc.
Licensed under the Creative Commons Attribution 4.0 International Public License.
HTML
diff --git a/lib/docs/scrapers/react_router.rb b/lib/docs/scrapers/react_router.rb
new file mode 100644
index 00000000..3d04d0fe
--- /dev/null
+++ b/lib/docs/scrapers/react_router.rb
@@ -0,0 +1,26 @@
+module Docs
+ class ReactRouter < UrlScraper
+ self.name = 'React Router'
+ self.slug = 'react_router'
+ self.type = 'simple'
+ self.release = '6.3.0'
+ self.base_url = 'https://reactrouterdotcom.fly.dev/docs/en/v6/api'
+
+ self.links = {
+ home: 'https://reactrouterdotcom.fly.dev/',
+ code: 'https://github.com/remix-run/react-router'
+ }
+
+ html_filters.push 'react_router/entries', 'react_router/clean_html', 'title'
+
+ options[:attribution] = <<-HTML
+ © React Training 2015-2019
+ © Remix Software 2020-2022
+ Licensed under the MIT License (MIT).
+ HTML
+
+ def get_latest_version(opts)
+ get_npm_version('react-router', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/redux.rb b/lib/docs/scrapers/redux.rb
index 31a8de84..4ef1a527 100644
--- a/lib/docs/scrapers/redux.rb
+++ b/lib/docs/scrapers/redux.rb
@@ -14,12 +14,12 @@ module Docs
options[:container] = '.markdown'
options[:attribution] = <<-HTML
- © 2015–2020 Dan Abramov
+ © 2015–2022 Dan Abramov
Licensed under the MIT License.
HTML
version do
- self.release = '4.0.5'
+ self.release = '4.2.0'
end
version '3' do
diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb
index 8c5f944d..74cd8ddf 100644
--- a/lib/docs/scrapers/rust.rb
+++ b/lib/docs/scrapers/rust.rb
@@ -3,7 +3,7 @@
module Docs
class Rust < UrlScraper
self.type = 'rust'
- self.release = '1.52.1'
+ self.release = '1.61.0'
self.base_url = 'https://doc.rust-lang.org/'
self.root_path = 'book/index.html'
self.initial_paths = %w(
diff --git a/lib/docs/scrapers/rxjs.rb b/lib/docs/scrapers/rxjs.rb
index 0c41a072..69d2fffc 100644
--- a/lib/docs/scrapers/rxjs.rb
+++ b/lib/docs/scrapers/rxjs.rb
@@ -4,7 +4,7 @@ module Docs
class Rxjs < UrlScraper
self.name = 'RxJS'
self.type = 'rxjs'
- self.release = '7.0.1'
+ self.release = '7.5.5'
self.base_url = 'https://rxjs.dev/'
self.root_path = 'guide/overview'
self.links = {
@@ -27,7 +27,7 @@ module Docs
options[:max_image_size] = 256_000
options[:attribution] = <<-HTML
- © 2015–2021 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors.
+ © 2015–2022 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors.
Code licensed under an Apache-2.0 License. Documentation licensed under CC BY 4.0.
HTML
diff --git a/lib/docs/scrapers/scala.rb b/lib/docs/scrapers/scala.rb
index dc268960..dcf28b63 100644
--- a/lib/docs/scrapers/scala.rb
+++ b/lib/docs/scrapers/scala.rb
@@ -3,24 +3,50 @@ module Docs
self.name = 'Scala'
self.type = 'scala'
self.links = {
- home: 'http://www.scala-lang.org/',
+ home: 'https://www.scala-lang.org/',
code: 'https://github.com/scala/scala'
}
- options[:container] = '#content-container'
options[:attribution] = <<-HTML
- © 2002-2019 EPFL, with contributions from Lightbend.
+ © 2002-2022 EPFL, with contributions from Lightbend.
Licensed under the Apache License, Version 2.0.
HTML
+ # For Scala 3, there is no official download link for the documentation
+ # (see https://contributors.scala-lang.org/t/5537).
+ #
+ # We currently need to build the docs ourselves. To do so:
+ # 1. Make sure that Scala 3 and sbt are installed
+ # (https://www.scala-lang.org/download/scala3.html)
+ # 2. Clone the Scala 3 (Dotty) repository (https://github.com/lampepfl/dotty)
+ # 3. From the Dotty folder, run this command in the terminal:
+ # $ sbt scaladoc/generateScalaDocumentation
+ # 4. Extract scaladoc/output/scala3/api/ into docs/scala~3.1
+ version '3.1' do
+ self.release = '3.1.1'
+ self.base_url = 'https://scala-lang.org/api/3.1.1/'
+ self.root_path = 'index.html'
+
+ options[:skip_patterns] = [
+ # Ignore class names with include “#”, which cause issues with the scraper
+ /%23/,
+
+ # Ignore local links to the Java documentation created by a Scaladoc bug
+ /java\/lang/,
+ ]
+
+ html_filters.push 'scala/entries_v3', 'scala/clean_html_v3'
+ end
+
# https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
# Extract api/scala-library into docs/scala~2.13_library
version '2.13 Library' do
self.release = '2.13.0'
self.base_url = 'https://www.scala-lang.org/api/2.13.0/'
self.root_path = 'index.html'
+ options[:container] = '#content-container'
- html_filters.push 'scala/entries', 'scala/clean_html'
+ html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
end
# https://downloads.lightbend.com/scala/2.13.0/scala-docs-2.13.0.zip
@@ -29,8 +55,9 @@ module Docs
self.release = '2.13.0'
self.base_url = 'https://www.scala-lang.org/api/2.13.0/scala-reflect/'
self.root_path = 'index.html'
+ options[:container] = '#content-container'
- html_filters.push 'scala/entries', 'scala/clean_html'
+ html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
end
# https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip
@@ -39,8 +66,9 @@ module Docs
self.release = '2.12.9'
self.base_url = 'https://www.scala-lang.org/api/2.12.9/'
self.root_path = 'index.html'
+ options[:container] = '#content-container'
- html_filters.push 'scala/entries', 'scala/clean_html'
+ html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
end
# https://downloads.lightbend.com/scala/2.12.9/scala-docs-2.12.9.zip
@@ -49,13 +77,14 @@ module Docs
self.release = '2.12.9'
self.base_url = 'https://www.scala-lang.org/api/2.12.9/scala-reflect/'
self.root_path = 'index.html'
+ options[:container] = '#content-container'
- html_filters.push 'scala/entries', 'scala/clean_html'
+ html_filters.push 'scala/entries_v2', 'scala/clean_html_v2'
end
def get_latest_version(opts)
- doc = fetch_doc('https://www.scala-lang.org/api/current/', opts)
- doc.at_css('#doc-version').content
+ doc = fetch_doc('https://www.scala-lang.org/api/3.x/', opts)
+ doc.at_css('.projectVersion').content
end
end
end
diff --git a/lib/docs/scrapers/sinon.rb b/lib/docs/scrapers/sinon.rb
index 055d62ab..a8a75f2d 100644
--- a/lib/docs/scrapers/sinon.rb
+++ b/lib/docs/scrapers/sinon.rb
@@ -27,6 +27,11 @@ module Docs
url
end
+ version '11' do
+ self.release = '11.1.2'
+ self.base_url = "https://sinonjs.org/releases/v#{release}/"
+ end
+
version '10' do
self.release = '10.0.1'
self.base_url = "https://sinonjs.org/releases/v#{release}/"
diff --git a/lib/docs/scrapers/socketio.rb b/lib/docs/scrapers/socketio.rb
index 34a88cc3..ec1f0f71 100644
--- a/lib/docs/scrapers/socketio.rb
+++ b/lib/docs/scrapers/socketio.rb
@@ -19,7 +19,7 @@ module Docs
HTML
version '4' do
- self.release = '4.0.0'
+ self.release = '4.1.2'
self.base_url = "https://socket.io/docs/v#{version}"
end
diff --git a/lib/docs/scrapers/spring_boot.rb b/lib/docs/scrapers/spring_boot.rb
index 9e9cbc33..6cec5faf 100644
--- a/lib/docs/scrapers/spring_boot.rb
+++ b/lib/docs/scrapers/spring_boot.rb
@@ -13,14 +13,13 @@ module Docs
options[:skip_patterns] = [/legal/]
+ # https://github.com/spring-projects/spring-boot/blob/main/buildSrc/src/main/resources/NOTICE.txt
options[:attribution] = <<-HTML
- Copyright © 2002–2020 Pivotal, Inc. All Rights Reserved.
+ Copyright © 2002–2022 Pivotal, Inc. All Rights Reserved.
HTML
- version do
- self.release = '2.4.0'
- self.base_url = "https://docs.spring.io/spring-boot/docs/#{release}/reference/html/"
- end
+ self.release = '2.7.0'
+ self.base_url = "https://docs.spring.io/spring-boot/docs/#{release}/reference/html/"
def get_latest_version(opts)
get_latest_github_release('spring-projects', 'spring-boot', opts)
diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb
index 1c82bd3a..854eb4a9 100644
--- a/lib/docs/scrapers/sqlite.rb
+++ b/lib/docs/scrapers/sqlite.rb
@@ -2,7 +2,7 @@ module Docs
class Sqlite < FileScraper
self.name = 'SQLite'
self.type = 'sqlite'
- self.release = '3.35.0'
+ self.release = '3.38.0'
self.base_url = 'https://sqlite.org/'
self.root_path = 'docs.html'
self.initial_paths = %w(keyword_index.html)
diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb
index b6764edd..450582e9 100644
--- a/lib/docs/scrapers/support_tables.rb
+++ b/lib/docs/scrapers/support_tables.rb
@@ -7,7 +7,7 @@ module Docs
self.name = 'Support Tables'
self.slug = 'browser_support_tables'
self.type = 'support_tables'
- self.release = '1.0.30001204'
+ self.release = '1.0.30001327'
def build_pages
url = 'https://github.com/Fyrd/caniuse/raw/main/data.json'
diff --git a/lib/docs/scrapers/tailwindcss.rb b/lib/docs/scrapers/tailwindcss.rb
new file mode 100644
index 00000000..408b99bf
--- /dev/null
+++ b/lib/docs/scrapers/tailwindcss.rb
@@ -0,0 +1,59 @@
+module Docs
+ class Tailwindcss < UrlScraper
+ self.name = 'Tailwind CSS'
+ self.type = 'tailwindcss'
+ self.slug = 'tailwindcss'
+ self.base_url = 'https://tailwindcss.com/docs'
+ self.root_path = '/'
+ self.release = '3.0.23'
+ self.links = {
+ home: 'tps://tailwindcss.com/',
+ code: 'https://github.com/tailwindlabs/tailwindcss'
+ }
+
+ html_filters.push 'tailwindcss/entries', 'tailwindcss/clean_html'
+
+ # Disable the clean text filter which removes empty nodes - we'll do it ourselves more selectively
+ text_filters.replace("clean_text", "tailwindcss/noop")
+
+ # Fix redirects from older tailwind 2 docs
+ options[:fix_urls] = lambda do |url|
+ if url.include? "installation/"
+ break "/docs/installation"
+ end
+
+ if url.end_with? "/breakpoints"
+ break "/docs/screens#{/#.*$/.match(url)}"
+ end
+ if url.end_with? "/adding-base-styles"
+ break "/docs/adding-custom-styles#adding-base-styles"
+ end
+ if url.end_with? "/ring-opacity"
+ break "/docs/ring-color#changing-the-opacity"
+ end
+
+ if url.match(/\/colors#?/)
+ break "/docs/customizing-colors#{/#.*$/.match(url)}"
+ end
+ end
+
+ options[:skip_patterns] = [
+ # Skip setup instructions
+ /\/browser-support$/,
+ /\/editor-setup$/,
+ /\/installation$/,
+ /\/optimizing-for-production$/,
+ /\/upgrade-guide/,
+ /\/using-with-preprocessors/
+ ]
+
+ #Obtainable from https://github.com/tailwindlabs/tailwindcss/blob/master/LICENSE
+ options[:attribution] = <<-HTML
+ © 2022 Tailwind Labs Inc.
+ HTML
+
+ def get_latest_version(opts)
+ get_latest_github_release('tailwindlabs', 'tailwindcss', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/tensorflow/tensorflow.rb b/lib/docs/scrapers/tensorflow/tensorflow.rb
index 4e1c4777..fbd68abc 100644
--- a/lib/docs/scrapers/tensorflow/tensorflow.rb
+++ b/lib/docs/scrapers/tensorflow/tensorflow.rb
@@ -14,11 +14,16 @@ module Docs
options[:container] = '.devsite-main-content'
options[:attribution] = <<-HTML
- © 2020 The TensorFlow Authors. All rights reserved.
+ © 2022 The TensorFlow Authors. All rights reserved.
Licensed under the Creative Commons Attribution License 4.0.
Code samples licensed under the Apache 2.0 License.
HTML
+ version '2.9' do
+ self.release = "2.9.1"
+ self.base_url = "https://www.tensorflow.org/versions/r#{version}/api_docs/python/tf"
+ end
+
version '2.4' do
self.release = "#{version}.0"
self.base_url = "https://www.tensorflow.org/versions/r#{version}/api_docs/python/tf"
diff --git a/lib/docs/scrapers/tensorflow/tensorflow_cpp.rb b/lib/docs/scrapers/tensorflow/tensorflow_cpp.rb
index feda35fd..e7dfbe9e 100644
--- a/lib/docs/scrapers/tensorflow/tensorflow_cpp.rb
+++ b/lib/docs/scrapers/tensorflow/tensorflow_cpp.rb
@@ -3,6 +3,11 @@ module Docs
self.name = 'TensorFlow C++'
self.slug = 'tensorflow_cpp'
+ version '2.9' do
+ self.release = "2.9.1"
+ self.base_url = "https://www.tensorflow.org/versions/r#{version}/api_docs/cc"
+ end
+
version '2.4' do
self.release = "#{version}.0"
self.base_url = "https://www.tensorflow.org/versions/r#{version}/api_docs/cc"
diff --git a/lib/docs/scrapers/terraform.rb b/lib/docs/scrapers/terraform.rb
index 0965ad06..411af235 100644
--- a/lib/docs/scrapers/terraform.rb
+++ b/lib/docs/scrapers/terraform.rb
@@ -20,8 +20,8 @@ module Docs
HTML
def get_latest_version(opts)
- contents = get_github_file_contents('hashicorp', 'terraform-website', 'content/config.rb', opts)
- contents.scan(/version\s+=\s+"([0-9.]+)"/)[0][0]
+ contents = get_latest_github_release('hashicorp', 'terraform', opts)
+ contents.sub("v", "")
end
end
end
diff --git a/lib/docs/scrapers/typescript.rb b/lib/docs/scrapers/typescript.rb
index dbc6dab6..ff2b39f4 100644
--- a/lib/docs/scrapers/typescript.rb
+++ b/lib/docs/scrapers/typescript.rb
@@ -4,7 +4,7 @@ module Docs
self.name = 'TypeScript'
self.type = 'typescript'
- self.release = '4.2.3'
+ self.release = '4.7.2'
self.base_urls = [
'https://www.typescriptlang.org/docs/handbook/',
'https://www.typescriptlang.org/'
@@ -30,11 +30,12 @@ module Docs
options[:skip_patterns] = [
/release-notes/,
+ /dt\/search/,
/play\//
]
options[:attribution] = <<-HTML
- © 2012-2021 Microsoft
+ © 2012-2022 Microsoft
Licensed under the Apache License, Version 2.0.
HTML
diff --git a/lib/docs/scrapers/underscore.rb b/lib/docs/scrapers/underscore.rb
index 00004759..1c517bc9 100644
--- a/lib/docs/scrapers/underscore.rb
+++ b/lib/docs/scrapers/underscore.rb
@@ -3,7 +3,7 @@ module Docs
self.name = 'Underscore.js'
self.slug = 'underscore'
self.type = 'underscore'
- self.release = '1.12.0'
+ self.release = '1.13.1'
self.base_url = 'https://underscorejs.org'
self.links = {
home: 'https://underscorejs.org',
@@ -17,7 +17,7 @@ module Docs
options[:skip_links] = true
options[:attribution] = <<-HTML
- © 2009–2020 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
+ © 2009–2021 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors
Licensed under the MIT License.
HTML
diff --git a/lib/docs/scrapers/vite.rb b/lib/docs/scrapers/vite.rb
new file mode 100644
index 00000000..15758a21
--- /dev/null
+++ b/lib/docs/scrapers/vite.rb
@@ -0,0 +1,27 @@
+module Docs
+ class Vite < UrlScraper
+ self.name = 'Vite'
+ self.slug = 'vite'
+ self.type = 'simple'
+ self.links = {
+ home: 'https://vitejs.dev/',
+ code: 'https://github.com/vitejs/vite'
+ }
+
+ options[:root_title] = 'Vite'
+
+ options[:attribution] = <<-HTML
+ © 2019–present, Yuxi (Evan) You and Vite contributors
+ Licensed under the MIT License.
+ HTML
+
+ self.release = '2.9.1'
+ self.base_url = 'https://vitejs.dev/'
+ self.initial_paths = %w(guide/)
+ html_filters.push 'vite/entries', 'vite/clean_html'
+
+ def get_latest_version(opts)
+ get_npm_version('vite', opts)
+ end
+ end
+end
diff --git a/lib/docs/scrapers/vue.rb b/lib/docs/scrapers/vue.rb
index 00acffbb..dd0a94a5 100644
--- a/lib/docs/scrapers/vue.rb
+++ b/lib/docs/scrapers/vue.rb
@@ -5,11 +5,12 @@ module Docs
self.type = 'vue'
self.links = {
home: 'https://vuejs.org/',
- code: 'https://github.com/vuejs/vue'
+ code: 'https://github.com/vuejs/core'
}
- options[:only_patterns] = [/guide\//, /api\//]
+ options[:only_patterns] = [/^$/ ,/guide\//, /api\//]
options[:skip] = %w(guide/team.html)
+ options[:skip_patterns] = [/guide\/contributing/]
options[:replace_paths] = { 'guide/' => 'guide/index.html' }
options[:attribution] = <<-HTML
@@ -18,16 +19,15 @@ module Docs
HTML
version '3' do
- self.release = '3.0.11'
- self.base_url = 'https://v3.vuejs.org/'
- self.root_path = 'guide/introduction.html'
- self.initial_paths = %w(api/)
+ self.release = '3.2.31'
+ self.base_url = 'https://vuejs.org/'
+ self.initial_paths = %w(guide/introduction.html)
html_filters.push 'vue/entries_v3', 'vue/clean_html'
end
version '2' do
self.release = '2.6.12'
- self.base_url = 'https://vuejs.org/v2/'
+ self.base_url = 'https://v2.vuejs.org/'
self.root_path = 'guide/index.html'
self.initial_paths = %w(api/)
html_filters.push 'vue/entries', 'vue/clean_html'
@@ -42,7 +42,7 @@ module Docs
end
def get_latest_version(opts)
- get_npm_version('vue', opts, 'next')
+ get_npm_version('vue', opts)
end
end
end
diff --git a/lib/docs/scrapers/vue_router.rb b/lib/docs/scrapers/vue_router.rb
index accb8455..3456fd59 100644
--- a/lib/docs/scrapers/vue_router.rb
+++ b/lib/docs/scrapers/vue_router.rb
@@ -21,7 +21,7 @@ module Docs
HTML
version '4' do
- self.release = '4.0.3'
+ self.release = '4.0.12'
self.base_url = 'https://next.router.vuejs.org/'
end
diff --git a/lib/docs/scrapers/vuex.rb b/lib/docs/scrapers/vuex.rb
index e2d9c66a..041593fb 100644
--- a/lib/docs/scrapers/vuex.rb
+++ b/lib/docs/scrapers/vuex.rb
@@ -19,7 +19,7 @@ module Docs
HTML
version '4' do
- self.release = '4.0.0'
+ self.release = '4.0.2'
self.base_url = 'https://next.vuex.vuejs.org/'
end
diff --git a/lib/docs/scrapers/webpack.rb b/lib/docs/scrapers/webpack.rb
index 526c47b5..3874970f 100644
--- a/lib/docs/scrapers/webpack.rb
+++ b/lib/docs/scrapers/webpack.rb
@@ -36,7 +36,7 @@ module Docs
HTML
version '5' do
- self.release = '5.37.0'
+ self.release = '5.72.0'
self.base_url = 'https://webpack.js.org/'
end
diff --git a/lib/docs/scrapers/werkzeug.rb b/lib/docs/scrapers/werkzeug.rb
index 905ee8a9..b55559f6 100755
--- a/lib/docs/scrapers/werkzeug.rb
+++ b/lib/docs/scrapers/werkzeug.rb
@@ -13,10 +13,15 @@ module Docs
options[:skip] = %w(changes/)
options[:attribution] = <<-HTML
- © 2007–2021 Pallets
+ © 2007–2022 Pallets
Licensed under the BSD 3-clause License.
HTML
+ version '2.1' do
+ self.release = '2.1.x'
+ self.base_url = "https://werkzeug.palletsprojects.com/en/#{self.release}/"
+ end
+
version '2.0' do
self.release = '2.0.x'
self.base_url = "https://werkzeug.palletsprojects.com/en/#{self.release}/"
diff --git a/lib/docs/scrapers/wordpress.rb b/lib/docs/scrapers/wordpress.rb
index 5c8939b5..ffb95a53 100644
--- a/lib/docs/scrapers/wordpress.rb
+++ b/lib/docs/scrapers/wordpress.rb
@@ -2,7 +2,7 @@ module Docs
class Wordpress < UrlScraper
self.name = 'WordPress'
self.type = 'wordpress'
- self.release = '5.7'
+ self.release = '6.0'
self.base_url = 'https://developer.wordpress.org/reference/'
self.initial_paths = %w(
functions/
@@ -32,7 +32,7 @@ module Docs
]
options[:attribution] = <<-HTML
- © 2003–2021 WordPress Foundation
+ © 2003–2022 WordPress Foundation
Licensed under the GNU GPLv2+ License.
HTML
diff --git a/lib/docs/scrapers/yarn.rb b/lib/docs/scrapers/yarn.rb
index 64c32180..8cc49260 100644
--- a/lib/docs/scrapers/yarn.rb
+++ b/lib/docs/scrapers/yarn.rb
@@ -1,28 +1,42 @@
module Docs
class Yarn < UrlScraper
self.type = 'simple'
- self.release = '1.19.0'
- self.base_url = 'https://yarnpkg.com/en/docs/'
- self.links = {
- home: 'https://yarnpkg.com/',
- code: 'https://github.com/yarnpkg/yarn'
- }
-
- html_filters.push 'yarn/entries', 'yarn/clean_html', 'title'
options[:root_title] = 'Yarn'
options[:trailing_slash] = false
options[:skip] = %w(nightly)
- options[:skip_patterns] = [/\Aorg\//]
options[:attribution] = <<-HTML
© 2016–present Yarn Contributors
Licensed under the BSD License.
HTML
+ version 'Berry' do
+ self.release = '3.1.1'
+ self.base_url = 'https://yarnpkg.com/'
+ self.links = {
+ home: 'https://yarnpkg.com/',
+ code: 'https://github.com/yarnpkg/berry'
+ }
+ html_filters.push 'yarn/entries_berry', 'yarn/clean_html_berry', 'title'
+ options[:skip] = ['features', 'cli', 'configuration', 'advanced']
+ options[:skip_patterns] = [/\Aapi/, /\Apackage/]
+ end
+
+ version 'Classic' do
+ self.release = '1.22.17'
+ self.base_url = 'https://classic.yarnpkg.com/en/docs/'
+ self.links = {
+ home: 'https://classic.yarnpkg.com/',
+ code: 'https://github.com/yarnpkg/yarn'
+ }
+ html_filters.push 'yarn/entries', 'yarn/clean_html', 'title'
+ options[:skip_patterns] = [/\Aorg\//]
+ end
+
def get_latest_version(opts)
- get_latest_github_release('yarnpkg', 'yarn', opts)
+ get_latest_github_release('yarnpkg', 'berry', opts)[/[\d.]+/]
end
end
end
diff --git a/lib/docs/scrapers/zig.rb b/lib/docs/scrapers/zig.rb
new file mode 100644
index 00000000..004b8757
--- /dev/null
+++ b/lib/docs/scrapers/zig.rb
@@ -0,0 +1,24 @@
+module Docs
+ class Zig < UrlScraper
+ self.name = 'Zig'
+ self.type = 'simple'
+ self.release = '0.9.0'
+ self.base_url = 'https://ziglang.org/documentation/0.9.0/'
+ self.links = {
+ home: 'https://ziglang.org/',
+ code: 'https://github.com/ziglang/zig'
+ }
+
+ html_filters.push 'zig/entries', 'zig/clean_html'
+
+ options[:follow_links] = false
+ options[:attribution] = <<-HTML
+ © 2015–2021, Zig contributors
+ HTML
+
+ def get_latest_version(opts)
+ tags = get_github_tags('ziglang', 'zig', opts)
+ tags[0]['name']
+ end
+ end
+end
diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor
index d63ea170..104c2da9 100644
--- a/lib/tasks/docs.thor
+++ b/lib/tasks/docs.thor
@@ -338,7 +338,7 @@ class DocsCLI < Thor
def download_doc(doc)
target_path = File.join(Docs.store_path, doc.path)
- open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
+ URI.open "https://downloads.devdocs.io/#{doc.path}.tar.gz" do |file|
FileUtils.mkpath(target_path)
file.close
tar = UnixUtils.gunzip(file.path)
diff --git a/public/icons/docs/codeigniter/SOURCE b/public/icons/docs/codeigniter/SOURCE
index dd068ebf..fc294337 100644
--- a/public/icons/docs/codeigniter/SOURCE
+++ b/public/icons/docs/codeigniter/SOURCE
@@ -1 +1 @@
-http://www.codeigniter.com/assets/data/ci-logo.zip
+https://www.codeigniter.com/data/ci-logo.zip
diff --git a/public/icons/docs/coffeescript/SOURCE b/public/icons/docs/coffeescript/SOURCE
index ba74eca4..2f97fe85 100644
--- a/public/icons/docs/coffeescript/SOURCE
+++ b/public/icons/docs/coffeescript/SOURCE
@@ -1 +1 @@
-https://github.com/jashkenas/coffee-script/downloads
+https://github.com/jashkenas/coffeescript/blob/master/docs/favicon-32x32.png
diff --git a/public/icons/docs/deno/16.png b/public/icons/docs/deno/16.png
new file mode 100644
index 00000000..f3ea5dd9
Binary files /dev/null and b/public/icons/docs/deno/16.png differ
diff --git a/public/icons/docs/deno/16@2x.png b/public/icons/docs/deno/16@2x.png
new file mode 100644
index 00000000..11d73732
Binary files /dev/null and b/public/icons/docs/deno/16@2x.png differ
diff --git a/public/icons/docs/deno/SOURCE b/public/icons/docs/deno/SOURCE
new file mode 100644
index 00000000..66a07585
--- /dev/null
+++ b/public/icons/docs/deno/SOURCE
@@ -0,0 +1 @@
+https://deno.land/logo.svg
diff --git a/public/icons/docs/eigen3/16.png b/public/icons/docs/eigen3/16.png
new file mode 100644
index 00000000..265a2f49
Binary files /dev/null and b/public/icons/docs/eigen3/16.png differ
diff --git a/public/icons/docs/eigen3/16@2x.png b/public/icons/docs/eigen3/16@2x.png
new file mode 100644
index 00000000..9cc98d2c
Binary files /dev/null and b/public/icons/docs/eigen3/16@2x.png differ
diff --git a/public/icons/docs/eigen3/SOURCE b/public/icons/docs/eigen3/SOURCE
new file mode 100644
index 00000000..439adb72
--- /dev/null
+++ b/public/icons/docs/eigen3/SOURCE
@@ -0,0 +1 @@
+https://gitlab.com/libeigen/eigen/-/blob/master/doc/Eigen_Silly_Professor_64x64.png
diff --git a/public/icons/docs/esbuild/16.png b/public/icons/docs/esbuild/16.png
new file mode 100644
index 00000000..c2891b01
Binary files /dev/null and b/public/icons/docs/esbuild/16.png differ
diff --git a/public/icons/docs/esbuild/16@2x.png b/public/icons/docs/esbuild/16@2x.png
new file mode 100644
index 00000000..9b233f85
Binary files /dev/null and b/public/icons/docs/esbuild/16@2x.png differ
diff --git a/public/icons/docs/esbuild/SOURCE b/public/icons/docs/esbuild/SOURCE
new file mode 100644
index 00000000..721ff1f5
--- /dev/null
+++ b/public/icons/docs/esbuild/SOURCE
@@ -0,0 +1 @@
+https://esbuild.github.io/favicon.svg
diff --git a/public/icons/docs/gnu_make/16.png b/public/icons/docs/gnu_make/16.png
new file mode 100644
index 00000000..a94aef6e
Binary files /dev/null and b/public/icons/docs/gnu_make/16.png differ
diff --git a/public/icons/docs/gnu_make/16@2x.png b/public/icons/docs/gnu_make/16@2x.png
new file mode 100644
index 00000000..6df3c03d
Binary files /dev/null and b/public/icons/docs/gnu_make/16@2x.png differ
diff --git a/public/icons/docs/gnu_make/SOURCE b/public/icons/docs/gnu_make/SOURCE
new file mode 100644
index 00000000..db0f6512
--- /dev/null
+++ b/public/icons/docs/gnu_make/SOURCE
@@ -0,0 +1 @@
+https://www.gnu.org/graphics/heckert_gnu.png
\ No newline at end of file
diff --git a/public/icons/docs/i3/16.png b/public/icons/docs/i3/16.png
new file mode 100644
index 00000000..d53f0127
Binary files /dev/null and b/public/icons/docs/i3/16.png differ
diff --git a/public/icons/docs/i3/16@2x.png b/public/icons/docs/i3/16@2x.png
new file mode 100644
index 00000000..afec0e90
Binary files /dev/null and b/public/icons/docs/i3/16@2x.png differ
diff --git a/public/icons/docs/i3/SOURCE b/public/icons/docs/i3/SOURCE
new file mode 100644
index 00000000..7a8de2b5
--- /dev/null
+++ b/public/icons/docs/i3/SOURCE
@@ -0,0 +1 @@
+https://github.com/i3/i3.github.io/blob/master/logo.png
diff --git a/public/icons/docs/kubectl/16.png b/public/icons/docs/kubectl/16.png
new file mode 100644
index 00000000..0dccf912
Binary files /dev/null and b/public/icons/docs/kubectl/16.png differ
diff --git a/public/icons/docs/kubectl/16@2x.png b/public/icons/docs/kubectl/16@2x.png
new file mode 100644
index 00000000..2261e8ec
Binary files /dev/null and b/public/icons/docs/kubectl/16@2x.png differ
diff --git a/public/icons/docs/kubectl/SOURCE b/public/icons/docs/kubectl/SOURCE
new file mode 100644
index 00000000..cedbe271
--- /dev/null
+++ b/public/icons/docs/kubectl/SOURCE
@@ -0,0 +1 @@
+https://cncf-branding.netlify.app/projects/kubernetes/
\ No newline at end of file
diff --git a/public/icons/docs/kubernetes/16.png b/public/icons/docs/kubernetes/16.png
new file mode 100644
index 00000000..79abeb07
Binary files /dev/null and b/public/icons/docs/kubernetes/16.png differ
diff --git a/public/icons/docs/kubernetes/16@2x.png b/public/icons/docs/kubernetes/16@2x.png
new file mode 100644
index 00000000..9291009d
Binary files /dev/null and b/public/icons/docs/kubernetes/16@2x.png differ
diff --git a/public/icons/docs/kubernetes/SOURCE b/public/icons/docs/kubernetes/SOURCE
new file mode 100644
index 00000000..cedbe271
--- /dev/null
+++ b/public/icons/docs/kubernetes/SOURCE
@@ -0,0 +1 @@
+https://cncf-branding.netlify.app/projects/kubernetes/
\ No newline at end of file
diff --git a/public/icons/docs/nix/16.png b/public/icons/docs/nix/16.png
new file mode 100644
index 00000000..13ad550b
Binary files /dev/null and b/public/icons/docs/nix/16.png differ
diff --git a/public/icons/docs/nix/16@2x.png b/public/icons/docs/nix/16@2x.png
new file mode 100644
index 00000000..5c1d1368
Binary files /dev/null and b/public/icons/docs/nix/16@2x.png differ
diff --git a/public/icons/docs/nix/SOURCE b/public/icons/docs/nix/SOURCE
new file mode 100644
index 00000000..c2f6c583
--- /dev/null
+++ b/public/icons/docs/nix/SOURCE
@@ -0,0 +1 @@
+https://github.com/NixOS/nixos-homepage/blob/master/logo/nixos-logo-only-hires.png
diff --git a/public/icons/docs/node/SOURCE b/public/icons/docs/node/SOURCE
index 9e63e3e8..a7dd51b2 100644
--- a/public/icons/docs/node/SOURCE
+++ b/public/icons/docs/node/SOURCE
@@ -1 +1 @@
-https://drive.google.com/folderview?id=0B2SAGsHi4DjIcHB0cTZZcXFZNG8&usp=sharing
+https://github.com/nodejs/nodejs.org/blob/main/static/images/logo-hexagon.png
diff --git a/public/icons/docs/point_cloud_library/16.png b/public/icons/docs/point_cloud_library/16.png
new file mode 100644
index 00000000..e952d096
Binary files /dev/null and b/public/icons/docs/point_cloud_library/16.png differ
diff --git a/public/icons/docs/point_cloud_library/16@2x.png b/public/icons/docs/point_cloud_library/16@2x.png
new file mode 100644
index 00000000..bd59a658
Binary files /dev/null and b/public/icons/docs/point_cloud_library/16@2x.png differ
diff --git a/public/icons/docs/point_cloud_library/SOURCE b/public/icons/docs/point_cloud_library/SOURCE
new file mode 100644
index 00000000..180ad4e2
--- /dev/null
+++ b/public/icons/docs/point_cloud_library/SOURCE
@@ -0,0 +1 @@
+https://github.com/PointCloudLibrary/pcl/blob/master/pcl.png
diff --git a/public/icons/docs/prettier/16.png b/public/icons/docs/prettier/16.png
new file mode 100644
index 00000000..212eae80
Binary files /dev/null and b/public/icons/docs/prettier/16.png differ
diff --git a/public/icons/docs/prettier/16@2x.png b/public/icons/docs/prettier/16@2x.png
new file mode 100644
index 00000000..7c0f3cc0
Binary files /dev/null and b/public/icons/docs/prettier/16@2x.png differ
diff --git a/public/icons/docs/prettier/SOURCE b/public/icons/docs/prettier/SOURCE
new file mode 100644
index 00000000..3e340ee6
--- /dev/null
+++ b/public/icons/docs/prettier/SOURCE
@@ -0,0 +1 @@
+https://prettier.io/icon.png
diff --git a/public/icons/docs/r/16.png b/public/icons/docs/r/16.png
new file mode 100644
index 00000000..2b0c75da
Binary files /dev/null and b/public/icons/docs/r/16.png differ
diff --git a/public/icons/docs/r/16@2x.png b/public/icons/docs/r/16@2x.png
new file mode 100644
index 00000000..51854fc2
Binary files /dev/null and b/public/icons/docs/r/16@2x.png differ
diff --git a/public/icons/docs/r/SOURCE b/public/icons/docs/r/SOURCE
new file mode 100644
index 00000000..c02c1e37
--- /dev/null
+++ b/public/icons/docs/r/SOURCE
@@ -0,0 +1 @@
+https://svn.r-project.org/R/trunk/doc/html/Rlogo.svg
diff --git a/public/icons/docs/react_native/SOURCE b/public/icons/docs/react_native/SOURCE
index c1d5f7fb..65f9e914 100644
--- a/public/icons/docs/react_native/SOURCE
+++ b/public/icons/docs/react_native/SOURCE
@@ -1 +1 @@
-https://github.com/facebook/react-native/blob/gh-pages/img/favicon.png
+https://github.com/facebook/react-native-website/blob/main/website/static/docs/assets/favicon.png
diff --git a/public/icons/docs/react_router/16.png b/public/icons/docs/react_router/16.png
new file mode 100644
index 00000000..3f6a9fbb
Binary files /dev/null and b/public/icons/docs/react_router/16.png differ
diff --git a/public/icons/docs/react_router/16@2x.png b/public/icons/docs/react_router/16@2x.png
new file mode 100644
index 00000000..55ae36cd
Binary files /dev/null and b/public/icons/docs/react_router/16@2x.png differ
diff --git a/public/icons/docs/react_router/SOURCE b/public/icons/docs/react_router/SOURCE
new file mode 100644
index 00000000..b2605efd
--- /dev/null
+++ b/public/icons/docs/react_router/SOURCE
@@ -0,0 +1 @@
+https://reactrouterdotcom.fly.dev/favicon.ico
diff --git a/public/icons/docs/tailwindcss/16.png b/public/icons/docs/tailwindcss/16.png
new file mode 100644
index 00000000..bee80120
Binary files /dev/null and b/public/icons/docs/tailwindcss/16.png differ
diff --git a/public/icons/docs/tailwindcss/16@2x.png b/public/icons/docs/tailwindcss/16@2x.png
new file mode 100644
index 00000000..a7378f5b
Binary files /dev/null and b/public/icons/docs/tailwindcss/16@2x.png differ
diff --git a/public/icons/docs/tailwindcss/SOURCE b/public/icons/docs/tailwindcss/SOURCE
new file mode 100644
index 00000000..2f5932a8
--- /dev/null
+++ b/public/icons/docs/tailwindcss/SOURCE
@@ -0,0 +1 @@
+https://tailwindcss.com/favicons/favicon.ico
diff --git a/public/icons/docs/vite/16.png b/public/icons/docs/vite/16.png
new file mode 100644
index 00000000..90ffa786
Binary files /dev/null and b/public/icons/docs/vite/16.png differ
diff --git a/public/icons/docs/vite/16@2x.png b/public/icons/docs/vite/16@2x.png
new file mode 100644
index 00000000..375fb680
Binary files /dev/null and b/public/icons/docs/vite/16@2x.png differ
diff --git a/public/icons/docs/vite/SOURCE b/public/icons/docs/vite/SOURCE
new file mode 100644
index 00000000..f1fe46ee
--- /dev/null
+++ b/public/icons/docs/vite/SOURCE
@@ -0,0 +1 @@
+https://vitejs.dev/logo.svg
diff --git a/public/icons/docs/zig/16.png b/public/icons/docs/zig/16.png
new file mode 100644
index 00000000..67a96b5b
Binary files /dev/null and b/public/icons/docs/zig/16.png differ
diff --git a/public/icons/docs/zig/16@2x.png b/public/icons/docs/zig/16@2x.png
new file mode 100644
index 00000000..3b9d9ef2
Binary files /dev/null and b/public/icons/docs/zig/16@2x.png differ
diff --git a/public/icons/docs/zig/SOURCE b/public/icons/docs/zig/SOURCE
new file mode 100644
index 00000000..9b31c377
--- /dev/null
+++ b/public/icons/docs/zig/SOURCE
@@ -0,0 +1,2 @@
+https://github.com/ziglang/logo/blob/master/zig-favicon.png
+https://github.com/ziglang/logo/blob/master/zig-mark.svg
diff --git a/renovate.json b/renovate.json
new file mode 100644
index 00000000..e717dbd9
--- /dev/null
+++ b/renovate.json
@@ -0,0 +1,3 @@
+{
+ "extends": ["github>freecodecamp/renovate-config"]
+}
diff --git a/test/lib/docs/core/doc_test.rb b/test/lib/docs/core/doc_test.rb
index b188ff0d..3da1fcfc 100644
--- a/test/lib/docs/core/doc_test.rb
+++ b/test/lib/docs/core/doc_test.rb
@@ -390,12 +390,16 @@ class DocsDocTest < MiniTest::Spec
it "compares versions" do
instance = doc.versions.first.new
+ assert_equal "Up-to-date", instance.outdated_state('0.0.2', '0.0.3')
+ assert_equal "Outdated major version", instance.outdated_state('0.2', '0.3')
assert_equal 'Up-to-date', instance.outdated_state('1', '1')
assert_equal 'Up-to-date', instance.outdated_state('1.2', '1.2')
assert_equal 'Up-to-date', instance.outdated_state('1.2.2', '1.2.2')
assert_equal 'Up-to-date', instance.outdated_state('1.2.2', '1.2.3')
assert_equal "Outdated major version", instance.outdated_state('1', '2')
- assert_equal "Outdated minor version", instance.outdated_state('1.2', '1.3')
+ assert_equal "Up-to-date", instance.outdated_state('1.0.2', '1.0.3')
+ assert_equal "Outdated major version", instance.outdated_state('1.2', '1.3')
+ assert_equal "Outdated minor version", instance.outdated_state('2.2', '2.3')
assert_equal "Outdated major version", instance.outdated_state('9', '10')
assert_equal "Outdated major version", instance.outdated_state('99', '101')
assert_equal 'Up-to-date', instance.outdated_state('2006-01-02', '2006-01-03')
diff --git a/views/app.erb b/views/app.erb
index ffaa1bf3..13890a5b 100644
--- a/views/app.erb
+++ b/views/app.erb
@@ -25,6 +25,7 @@
Changelog
Guide
About
+ Report a bug