From e5fc6c2042511e0fb40f7cd18f2ff651bf713d83 Mon Sep 17 00:00:00 2001 From: Stefan Breunig Date: Fri, 14 Jun 2024 17:03:56 +0200 Subject: [PATCH 1/2] Update Elixir documentation (1.16 and 1.17) The guides have moved from a separate page into the same documentation system. This required shuffling of the cleanup functions, which is subtly incompatible with the docs of the older versions, or would have required if/else in various places. For legibility, I thus opted to remove old versions. --- lib/docs/filters/elixir/clean_html.rb | 29 ++--- lib/docs/filters/elixir/entries.rb | 38 +++--- lib/docs/scrapers/elixir.rb | 170 ++------------------------ 3 files changed, 32 insertions(+), 205 deletions(-) diff --git a/lib/docs/filters/elixir/clean_html.rb b/lib/docs/filters/elixir/clean_html.rb index c2c31d46..6783afb2 100644 --- a/lib/docs/filters/elixir/clean_html.rb +++ b/lib/docs/filters/elixir/clean_html.rb @@ -2,32 +2,12 @@ module Docs class Elixir class CleanHtmlFilter < Filter def call - if current_url.path.start_with?('/getting-started') - guide - else - api - end + api doc end - def guide - @doc = at_css('#content article') - - css('pre > code').each do |node| - node.parent.content = node.content - end - - css('div > pre.highlight').each do |node| - node.content = node.content - node['data-language'] = node.parent['class'][/language-(\w+)/, 1] - node.parent.before(node).remove - end - end - def api - css('.hover-link', 'footer', ':not(.detail-header) > .view-source').remove - - css('h1 .settings').remove + css('.top-search').remove css('.summary').each do |node| node.name = 'dl' @@ -65,6 +45,11 @@ module Docs end end + css('h1 a.icon-action[title="View Source"]').each do |node| + node['class'] = 'source' + node.content = "Source" + end + css('pre').each do |node| node['data-language'] = 'elixir' node.content = node.content diff --git a/lib/docs/filters/elixir/entries.rb b/lib/docs/filters/elixir/entries.rb index 7f73b54a..640f65e1 100644 --- a/lib/docs/filters/elixir/entries.rb +++ b/lib/docs/filters/elixir/entries.rb @@ -3,25 +3,21 @@ module Docs class EntriesFilter < Docs::EntriesFilter def get_name css('h1 .app-vsn').remove - name = (at_css('h1 > span') or at_css('h1')).content.strip - - if current_url.path.start_with?('/getting-started') - name.remove(/\.\z/) - else - name = name.split(' ').first unless name.start_with?('mix ') # ecto - name - end + (at_css('h1 > span') or at_css('h1')).content.strip end def get_type - if current_url.path.start_with?('/getting-started') - if subpath.start_with?('mix-otp') - 'Guide: Mix & OTP' - elsif subpath.start_with?('meta') - 'Guide: Metaprogramming' - else - 'Guide' - end + section = at_css('h1 a.source').attr('href').match('elixir/pages/([^/]+)/')&.captures&.first + if section == "mix-and-otp" + return "Mix & OTP" + elsif section + return section.gsub("-", " ").capitalize + end + + name = at_css('h1 span').text + case name.split(' ').first + when 'mix' then 'Mix Tasks' + when 'Changelog' then 'References' else case at_css('h1 small').try(:content) when 'exception' @@ -29,19 +25,13 @@ module Docs when 'protocol' 'Protocols' else - if name.start_with?('Phoenix') - name.split('.')[0..2].join('.') - elsif name.start_with?('mix ') - 'Mix Tasks' - else - name.split('.').first - end + name end end end def additional_entries - return [] if type == 'Exceptions' || type == 'Guide' || root_page? + return [] if root_page? css('.detail-header').map do |node| id = node['id'] diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index 75bee5e9..2523235e 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -4,7 +4,7 @@ module Docs self.name = 'Elixir' self.type = 'elixir' - self.root_path = 'api-reference.html' + self.root_path = 'introduction.html' self.links = { home: 'https://elixir-lang.org/', code: 'https://github.com/elixir-lang/elixir' @@ -12,193 +12,45 @@ module Docs html_filters.push 'elixir/clean_html', 'elixir/entries', 'title' - options[:container] = ->(filter) { - filter.current_url.path.start_with?('/getting-started') ? '#main' : '#content' - } + options[:container] = '#content' options[:title] = false options[:root_title] = 'Elixir' options[:attribution] = <<-HTML - © 2012 Plataformatec
+ © 2012 - 2024 The Elixir Team
Licensed under the Apache License, Version 2.0. HTML def initial_urls - [ "https://hexdocs.pm/elixir/#{self.class.release}/api-reference.html", + [ "https://hexdocs.pm/elixir/#{self.class.release}/introduction.html", "https://hexdocs.pm/eex/#{self.class.release}/EEx.html", "https://hexdocs.pm/ex_unit/#{self.class.release}/ExUnit.html", "https://hexdocs.pm/iex/#{self.class.release}/IEx.html", "https://hexdocs.pm/logger/#{self.class.release}/Logger.html", - "https://hexdocs.pm/mix/#{self.class.release}/Mix.html", - "https://elixir-lang.org/getting-started/introduction.html" ] - end - - version '1.15' do - self.release = '1.15.4' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.14' do - self.release = '1.14.1' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.13' do - self.release = '1.13.4' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.12' do - self.release = '1.12.0' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.11' do - self.release = '1.11.2' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.10' do - self.release = '1.10.4' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.9' do - self.release = '1.9.4' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.8' do - self.release = '1.8.2' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.7' do - self.release = '1.7.4' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.6' do - self.release = '1.6.6' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] - end - - version '1.5' do - self.release = '1.5.3' - self.base_urls = [ - "https://hexdocs.pm/elixir/#{release}/", - "https://hexdocs.pm/eex/#{release}/", - "https://hexdocs.pm/ex_unit/#{release}/", - "https://hexdocs.pm/iex/#{release}/", - "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' - ] + "https://hexdocs.pm/mix/#{self.class.release}/Mix.html" ] end - version '1.4' do - self.release = '1.4.5' + version '1.17' do + self.release = '1.17.0' self.base_urls = [ "https://hexdocs.pm/elixir/#{release}/", "https://hexdocs.pm/eex/#{release}/", "https://hexdocs.pm/ex_unit/#{release}/", "https://hexdocs.pm/iex/#{release}/", "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' + "https://hexdocs.pm/mix/#{release}/" ] end - version '1.3' do - self.release = '1.3.4' + version '1.16' do + self.release = '1.16.3' self.base_urls = [ "https://hexdocs.pm/elixir/#{release}/", "https://hexdocs.pm/eex/#{release}/", "https://hexdocs.pm/ex_unit/#{release}/", "https://hexdocs.pm/iex/#{release}/", "https://hexdocs.pm/logger/#{release}/", - "https://hexdocs.pm/mix/#{release}/", - 'https://elixir-lang.org/getting-started/' + "https://hexdocs.pm/mix/#{release}/" ] end From 59aaafcbbf71a595870d42d7b3ae6a00ffec9761 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Wed, 17 Jul 2024 06:38:49 +0200 Subject: [PATCH 2/2] Update Elixir documentation (1.17.2) --- lib/docs/scrapers/elixir.rb | 175 +++++++++++++++++++++++++++++++++++- 1 file changed, 173 insertions(+), 2 deletions(-) diff --git a/lib/docs/scrapers/elixir.rb b/lib/docs/scrapers/elixir.rb index 2523235e..47f361e6 100644 --- a/lib/docs/scrapers/elixir.rb +++ b/lib/docs/scrapers/elixir.rb @@ -17,7 +17,7 @@ module Docs options[:root_title] = 'Elixir' options[:attribution] = <<-HTML - © 2012 - 2024 The Elixir Team
+ © 2012-2024 The Elixir Team
Licensed under the Apache License, Version 2.0. HTML @@ -31,7 +31,7 @@ module Docs end version '1.17' do - self.release = '1.17.0' + self.release = '1.17.2' self.base_urls = [ "https://hexdocs.pm/elixir/#{release}/", "https://hexdocs.pm/eex/#{release}/", @@ -54,6 +54,177 @@ module Docs ] end + # scraping of older versions is no longer supported! + + version '1.15' do + self.release = '1.15.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.14' do + self.release = '1.14.1' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.13' do + self.release = '1.13.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.12' do + self.release = '1.12.0' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.11' do + self.release = '1.11.2' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.10' do + self.release = '1.10.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.9' do + self.release = '1.9.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.8' do + self.release = '1.8.2' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.7' do + self.release = '1.7.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.6' do + self.release = '1.6.6' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.5' do + self.release = '1.5.3' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.4' do + self.release = '1.4.5' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + + version '1.3' do + self.release = '1.3.4' + self.base_urls = [ + "https://hexdocs.pm/elixir/#{release}/", + "https://hexdocs.pm/eex/#{release}/", + "https://hexdocs.pm/ex_unit/#{release}/", + "https://hexdocs.pm/iex/#{release}/", + "https://hexdocs.pm/logger/#{release}/", + "https://hexdocs.pm/mix/#{release}/", + 'https://elixir-lang.org/getting-started/' + ] + end + def get_latest_version(opts) doc = fetch_doc('https://hexdocs.pm/elixir/api-reference.html', opts) doc.at_css('.sidebar-projectVersion').content.strip[1..-1]