diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8d2f066..a0159be9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Set up Ruby - uses: ruby/setup-ruby@161cd54b698f1fb3ea539faab2e036d409550e3c # v1.187.0 + uses: ruby/setup-ruby@50ba3386b050ad5b97a41fcb81240cbee1d1821f # v1.188.0 with: bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Run tests diff --git a/.github/workflows/schedule-doc-report.yml b/.github/workflows/schedule-doc-report.yml index e4488d80..30ab4418 100644 --- a/.github/workflows/schedule-doc-report.yml +++ b/.github/workflows/schedule-doc-report.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - name: Set up Ruby - uses: ruby/setup-ruby@161cd54b698f1fb3ea539faab2e036d409550e3c # v1.187.0 + uses: ruby/setup-ruby@50ba3386b050ad5b97a41fcb81240cbee1d1821f # v1.188.0 with: bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Generate report diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99ab59bb..d0df2d11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Ruby - uses: ruby/setup-ruby@161cd54b698f1fb3ea539faab2e036d409550e3c # v1.187.0 + uses: ruby/setup-ruby@50ba3386b050ad5b97a41fcb81240cbee1d1821f # v1.188.0 with: bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Run tests diff --git a/Gemfile b/Gemfile index cfbf549f..e3847952 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,6 @@ group :development do end group :docs do - gem 'net-sftp', require: false gem 'progress_bar', require: false gem 'redcarpet' gem 'tty-pager', require: false diff --git a/Gemfile.lock b/Gemfile.lock index e673c7b6..2b9d1714 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,9 +57,6 @@ GEM mustermann (3.0.0) ruby2_keywords (~> 0.0.1) mutex_m (0.2.0) - net-sftp (4.0.0) - net-ssh (>= 5.0.0, < 8.0.0) - net-ssh (7.0.1) newrelic_rpm (8.16.0) nokogiri (1.16.6) mini_portile2 (~> 2.8.2) @@ -159,7 +156,6 @@ DEPENDENCIES image_optim image_optim_pack minitest - net-sftp newrelic_rpm nokogiri progress_bar diff --git a/docs/file-scrapers.md b/docs/file-scrapers.md index 1a7149b3..9d51e4f4 100644 --- a/docs/file-scrapers.md +++ b/docs/file-scrapers.md @@ -196,10 +196,13 @@ mv ./usr/share/doc/openjdk-16-jre-headless/api/ docs/openjdk~$VERSION ## Pandas +From the home directory; `devdocs`, execute below: + ```sh -curl https://pandas.pydata.org/docs/pandas.zip | bsdtar --extract --file - --directory=docs/pandas~1 +curl https://pandas.pydata.org/docs/pandas.zip -o tmp.zip && unzip tmp.zip -d docs/pandas~2 && rm tmp.zip ``` + ## PHP Click the link under the "Many HTML files" column on https://www.php.net/download-docs.php, extract the tarball, change its name to `php` and put it in `docs/`. diff --git a/lib/docs/filters/elixir/clean_html.rb b/lib/docs/filters/elixir/clean_html.rb index c2c31d46..6783afb2 100644 --- a/lib/docs/filters/elixir/clean_html.rb +++ b/lib/docs/filters/elixir/clean_html.rb @@ -2,32 +2,12 @@ module Docs class Elixir class CleanHtmlFilter < Filter def call - if current_url.path.start_with?('/getting-started') - guide - else - api - end + api doc end - def guide - @doc = at_css('#content article') - - css('pre > code').each do |node| - node.parent.content = node.content - end - - css('div > pre.highlight').each do |node| - node.content = node.content - node['data-language'] = node.parent['class'][/language-(\w+)/, 1] - node.parent.before(node).remove - end - end - def api - css('.hover-link', 'footer', ':not(.detail-header) > .view-source').remove - - css('h1 .settings').remove + css('.top-search').remove css('.summary').each do |node| node.name = 'dl' @@ -65,6 +45,11 @@ module Docs end end + css('h1 a.icon-action[title="View Source"]').each do |node| + node['class'] = 'source' + node.content = "Source" + end + css('pre').each do |node| node['data-language'] = 'elixir' node.content = node.content diff --git a/lib/docs/filters/elixir/entries.rb b/lib/docs/filters/elixir/entries.rb index 7f73b54a..640f65e1 100644 --- a/lib/docs/filters/elixir/entries.rb +++ b/lib/docs/filters/elixir/entries.rb @@ -3,25 +3,21 @@ module Docs class EntriesFilter < Docs::EntriesFilter def get_name css('h1 .app-vsn').remove - name = (at_css('h1 > span') or at_css('h1')).content.strip - - if current_url.path.start_with?('/getting-started') - name.remove(/\.\z/) - else - name = name.split(' ').first unless name.start_with?('mix ') # ecto - name - end + (at_css('h1 > span') or at_css('h1')).content.strip end def get_type - if current_url.path.start_with?('/getting-started') - if subpath.start_with?('mix-otp') - 'Guide: Mix & OTP' - elsif subpath.start_with?('meta') - 'Guide: Metaprogramming' - else - 'Guide' - end + section = at_css('h1 a.source').attr('href').match('elixir/pages/([^/]+)/')&.captures&.first + if section == "mix-and-otp" + return "Mix & OTP" + elsif section + return section.gsub("-", " ").capitalize + end + + name = at_css('h1 span').text + case name.split(' ').first + when 'mix' then 'Mix Tasks' + when 'Changelog' then 'References' else case at_css('h1 small').try(:content) when 'exception' @@ -29,19 +25,13 @@ module Docs when 'protocol' 'Protocols' else - if name.start_with?('Phoenix') - name.split('.')[0..2].join('.') - elsif name.start_with?('mix ') - 'Mix Tasks' - else - name.split('.').first - end + name end end end def additional_entries - return [] if type == 'Exceptions' || type == 'Guide' || root_page? + return [] if root_page? css('.detail-header').map do |node| id = node['id'] diff --git a/lib/docs/filters/qunit/clean_html.rb b/lib/docs/filters/qunit/clean_html.rb index eb104b5c..ff4d9ec9 100644 --- a/lib/docs/filters/qunit/clean_html.rb +++ b/lib/docs/filters/qunit/clean_html.rb @@ -4,7 +4,7 @@ module Docs class Qunit class CleanHtmlFilter < Filter def call - @doc = at_css('.content[role="main"]') + @doc = at_css('.content[role="main"] > article') css('.sidebar').remove css('pre').each do |node| node['data-language'] = 'javascript' diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index 75bee5e9..47f361e6 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -4,7 +4,7 @@ module Docs self.name = 'Elixir' self.type = 'elixir' - self.root_path = 'api-reference.html' + self.root_path = 'introduction.html' self.links = { home: 'https://elixir-lang.org/', code: 'https://github.com/elixir-lang/elixir' @@ -12,27 +12,50 @@ module Docs html_filters.push 'elixir/clean_html', 'elixir/entries', 'title' - options[:container] = ->(filter) { - filter.current_url.path.start_with?('/getting-started') ? '#main' : '#content' - } + options[:container] = '#content' options[:title] = false options[:root_title] = 'Elixir' options[:attribution] = <<-HTML - © 2012 Plataformatec
+ © 2012-2024 The Elixir Team
Licensed under the Apache License, Version 2.0. HTML def initial_urls - [ "https://hexdocs.pm/elixir/#{self.class.release}/api-reference.html", + [ "https://hexdocs.pm/elixir/#{self.class.release}/introduction.html", "https://hexdocs.pm/eex/#{self.class.release}/EEx.html", "https://hexdocs.pm/ex_unit/#{self.class.release}/ExUnit.html", "https://hexdocs.pm/iex/#{self.class.release}/IEx.html", "https://hexdocs.pm/logger/#{self.class.release}/Logger.html", - "https://hexdocs.pm/mix/#{self.class.release}/Mix.html", - "https://elixir-lang.org/getting-started/introduction.html" ] + "https://hexdocs.pm/mix/#{self.class.release}/Mix.html" ] + end + + version '1.17' do + self.release = '1.17.2' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/" + ] end + version '1.16' do + self.release = '1.16.3' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/" + ] + end + + # scraping of older versions is no longer supported! + version '1.15' do self.release = '1.15.4' self.base_urls = [ diff --git a/lib/docs/scrapers/fastapi.rb b/lib/docs/scrapers/fastapi.rb index b19a1c78..ed857934 100644 --- a/lib/docs/scrapers/fastapi.rb +++ b/lib/docs/scrapers/fastapi.rb @@ -2,7 +2,7 @@ module Docs class Fastapi < UrlScraper self.name = 'FastAPI' self.type = 'fastapi' - self.release = '0.100.1' + self.release = '0.111.1' self.base_url = 'https://fastapi.tiangolo.com/' self.root_path = '/' self.links = { diff --git a/lib/docs/scrapers/laravel.rb b/lib/docs/scrapers/laravel.rb index 9b3c0771..03872ce0 100644 --- a/lib/docs/scrapers/laravel.rb +++ b/lib/docs/scrapers/laravel.rb @@ -29,8 +29,22 @@ module Docs Laravel is a trademark of Taylor Otwell. HTML + version '11' do + self.release = '11.11.1' + self.root_path = '/api/11.x/index.html' + self.initial_paths = %w(/docs/11.x/installation /api/11.x/classes.html) + + options[:only_patterns] = [%r{\A/api/11\.x/}, %r{\A/docs/11\.x/}] + + options[:fix_urls] = ->(url) do + url.sub! %r{11.x/+}, "11.x/" + url.sub! %r{#{Regexp.escape(Laravel.base_url)}/docs\/(?![1-9]?\d)}, "#{Laravel.base_url}/docs/11.x/" + url + end + end + version '10' do - self.release = '10.13.0' + self.release = '10.48.14' self.root_path = '/api/10.x/index.html' self.initial_paths = %w(/docs/10.x/installation /api/10.x/classes.html) @@ -44,7 +58,7 @@ module Docs end version '9' do - self.release = '9.52.8' + self.release = '9.52.16' self.root_path = '/api/9.x/index.html' self.initial_paths = %w(/docs/9.x/installation /api/9.x/classes.html) diff --git a/lib/docs/scrapers/pandas.rb b/lib/docs/scrapers/pandas.rb index d748c297..c95f7561 100644 --- a/lib/docs/scrapers/pandas.rb +++ b/lib/docs/scrapers/pandas.rb @@ -16,6 +16,29 @@ module Docs Licensed under the 3-clause BSD License. HTML + version '2' do + self.release = '2.2.2' + self.base_url = "https://pandas.pydata.org/pandas-docs/version/#{self.release}/" + + html_filters.push 'pandas/clean_html', 'pandas/entries' + + options[:container] = 'main section' + + options[:skip_patterns] = [ + /development/, + /getting_started/, + /whatsnew/ + ] + + options[:skip] = [ + 'panel.html', + 'pandas.pdf', + 'pandas.zip', + 'ecosystem.html' + ] + + end + version '1' do self.release = '1.5.0' self.base_url = "https://pandas.pydata.org/pandas-docs/version/#{self.release}/" diff --git a/lib/docs/scrapers/python.rb b/lib/docs/scrapers/python.rb index 3988bba3..507080d1 100644 --- a/lib/docs/scrapers/python.rb +++ b/lib/docs/scrapers/python.rb @@ -1,5 +1,5 @@ module Docs - class Python < FileScraper + class Python < UrlScraper self.type = 'python' self.root_path = 'index.html' self.links = { @@ -23,12 +23,12 @@ module Docs library/sunau.html) options[:attribution] = <<-HTML - © 2001–2023 Python Software Foundation
+ © 2001–2024 Python Software Foundation
Licensed under the PSF License. HTML version '3.12' do - self.release = '3.12.1' + self.release = '3.12.4' self.base_url = "https://docs.python.org/#{self.version}/" html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html' diff --git a/lib/docs/scrapers/qunit.rb b/lib/docs/scrapers/qunit.rb index f0d649f5..ff13fbdb 100644 --- a/lib/docs/scrapers/qunit.rb +++ b/lib/docs/scrapers/qunit.rb @@ -4,8 +4,8 @@ module Docs class Qunit < UrlScraper self.name = 'QUnit' self.type = 'qunit' - self.release = '2.19.3' - self.base_url = 'https://api.qunitjs.com/' + self.release = '2.21.0' + self.base_url = 'https://qunitjs.com/api/' self.root_path = '/' self.links = { home: 'https://qunitjs.com/', @@ -18,13 +18,14 @@ module Docs options[:container] = '.main' options[:skip_patterns] = [ - /deprecated/, /^QUnit$/, /^assert$/, /^callbacks$/, /^async$/, /^config$/, /^extension$/, + /^deprecated$/, + /^removed$/, ] options[:attribution] = <<-HTML diff --git a/lib/docs/scrapers/rust.rb b/lib/docs/scrapers/rust.rb index 536aaefd..e7d87ce0 100644 --- a/lib/docs/scrapers/rust.rb +++ b/lib/docs/scrapers/rust.rb @@ -3,7 +3,7 @@ module Docs class Rust < UrlScraper self.type = 'rust' - self.release = '1.75.0' + self.release = '1.79.0' self.base_url = 'https://doc.rust-lang.org/' self.root_path = 'book/index.html' self.initial_paths = %w( diff --git a/lib/docs/scrapers/support_tables.rb b/lib/docs/scrapers/support_tables.rb index 7d9570a9..4318edf3 100644 --- a/lib/docs/scrapers/support_tables.rb +++ b/lib/docs/scrapers/support_tables.rb @@ -7,7 +7,7 @@ module Docs self.name = 'Support Tables' self.slug = 'browser_support_tables' self.type = 'support_tables' - self.release = '1.0.30001574' + self.release = '1.0.30001642' self.base_url = 'https://github.com/Fyrd/caniuse/raw/main/' # https://github.com/Fyrd/caniuse/blob/main/LICENSE diff --git a/lib/docs/scrapers/vitest.rb b/lib/docs/scrapers/vitest.rb index 251b6c69..bbb82a27 100644 --- a/lib/docs/scrapers/vitest.rb +++ b/lib/docs/scrapers/vitest.rb @@ -17,7 +17,7 @@ module Docs Licensed under the MIT License. HTML - self.release = '1.2.2' + self.release = '2.0.3' self.base_url = 'https://vitest.dev/' self.initial_paths = %w(guide/) html_filters.push 'vitest/entries', 'vite/clean_html' diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor index 54009530..5b85a8b0 100644 --- a/lib/tasks/docs.thor +++ b/lib/tasks/docs.thor @@ -162,8 +162,6 @@ class DocsCLI < Thor option :dryrun, type: :boolean option :packaged, type: :boolean def upload(*names) - require 'net/sftp' - if options[:packaged] slugs = Dir[File.join(Docs.store_path, '*.tar.gz')].map { |f| File.basename(f, '.tar.gz') } docs = find_docs_by_slugs(slugs) diff --git a/techstack.md b/techstack.md index f9657f09..d6415425 100644 --- a/techstack.md +++ b/techstack.md @@ -159,7 +159,6 @@ Full tech stack [here](/techstack.md) |[image_optim](https://rubygems.org/image_optim)|v0.31.3|11/14/22|Paul Sernatinger |MIT|N/A| |[image_optim_pack](https://rubygems.org/image_optim_pack)|v0.10.1|11/14/22|Paul Sernatinger |MIT|N/A| |[minitest](https://rubygems.org/minitest)|v5.21.2|01/03/15|Thibaut |MIT|N/A| -|[net-sftp](https://rubygems.org/net-sftp)|v4.0.0|11/14/22|Paul Sernatinger |MIT|N/A| |[newrelic_rpm](https://rubygems.org/newrelic_rpm)|v8.16.0|03/24/18|Thibaut Courouble |Apache-2.0|N/A| |[nokogiri](https://rubygems.org/nokogiri)|v1.16.0|10/21/18|Thibaut Courouble |MIT|N/A| |[progress_bar](https://rubygems.org/progress_bar)|v1.3.3|01/26/14|Thibaut |WTFPL|N/A| diff --git a/techstack.yml b/techstack.yml index 88f89c51..99d14af4 100644 --- a/techstack.yml +++ b/techstack.yml @@ -268,20 +268,6 @@ tools: detection_source: Gemfile last_updated_by: Thibaut last_updated_on: 2015-01-03 15:38:22.000000000 Z -- name: net-sftp - description: A pure Ruby implementation of the SFTP client protocol - package_url: https://rubygems.org/net-sftp - version: 4.0.0 - license: MIT - open_source: true - hosted_saas: false - category: Libraries - sub_category: RubyGems Packages - image_url: https://img.stackshare.io/package/19106/default_92b412aaa5197ea1fc2d70a503018e4186714110.png - detection_source_url: https://github.com/freeCodeCamp/devdocs/blob/main/Gemfile.lock - detection_source: Gemfile - last_updated_by: Paul Sernatinger - last_updated_on: 2022-11-14 14:30:30.000000000 Z - name: newrelic_rpm description: New Relic is a performance management system, developed by New Relic, Inc