Move rails from FileScraper to URLscraper [initial steps]

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -738,7 +738,7 @@ credits = [
     'https://www.ruby-lang.org/en/about/license.txt'
   ], [
     'Ruby on Rails',
-    '2004-2020 David Heinemeier Hansson<br>Rails, Ruby on Rails, and the Rails logo are trademarks of David Heinemeier Hansson.',
+    '2004-2021 David Heinemeier Hansson<br>Rails, Ruby on Rails, and the Rails logo are trademarks of David Heinemeier Hansson.',
     'MIT',
     'https://raw.githubusercontent.com/rails/rails/master/activerecord/MIT-LICENSE'
   ], [

lib/docs/filters/rails/clean_html_guides.rb

@@ -2,7 +2,7 @@ module Docs
   class Rails
     class CleanHtmlGuidesFilter < Filter
       def call
-        return doc unless slug.start_with?('guides')
+        return doc unless root_url.to_s.match?('guides')
 
         at_css('#mainCol').prepend_child at_css('#feature .wrapper').children
         @doc = at_css('#mainCol')

lib/docs/filters/rails/entries.rb

@@ -1,6 +1,6 @@
 module Docs
   class Rails
-    class EntriesFilter < Docs::Rdoc::EntriesFilter
+    class EntriesFilter < Docs::EntriesFilter# Docs::Rdoc::EntriesFilter
       TYPE_BY_NAME_MATCHES = {
         /Assertions|::Test|Fixture/                          => 'Testing',
         /\AActiveRecord.+mysql/i                             => 'ActiveRecord/MySQL',
@@ -49,7 +49,7 @@ module Docs
       end
 
       def get_type
-        return 'Guides' if slug.start_with?('guides')
+        return 'Guides' if root_url.to_s.match?('guides')
 
         parent = at_css('.meta-parent').try(:content).to_s
 
@@ -69,13 +69,13 @@ module Docs
       end
 
       def include_default_entry?
-        return true if slug.start_with?('guides')
+        return true if root_url.to_s.match?('guides')
 
         super && !skip?
       end
 
       def additional_entries
-        return [] if slug.start_with?('guides')
+        return [] if root_url.to_s.match?('guides')
 
         skip? ? [] : super
       end

lib/docs/scrapers/rails.rb

@@ -0,0 +1,128 @@
+module Docs
+  class Rails < UrlScraper
+    # include FixInternalUrlsBehavior
+    include MultipleBaseUrls
+
+    self.name = 'Ruby on Rails'
+    self.type = 'rdoc'
+    self.slug = 'rails'
+
+    self.links = {
+      home: 'http://rubyonrails.org/',
+      code: 'https://github.com/rails/rails'
+    }
+
+    # html_filters.replace 'container', 'rails/container'
+    html_filters.push 'rails/entries', 'rdoc/clean_html', 'rails/clean_html_guides'
+
+    options[:skip_rdoc_filters?] = ->(filter) { filter.root_url.to_s.match?('guides/') }
+
+    options[:root_title] = 'Ruby on Rails'
+
+    options[:skip] = [
+      'links.html',
+      'index.html',
+      'credits.html',
+      'ruby_on_rails_guides_guidelines.html',
+      'contributing_to_ruby_on_rails.html',
+      'development_dependencies_install.html',
+      'api_documentation_guidelines.html',
+      'ActionController/Instrumentation.html',
+      'ActionController/Rendering.html',
+      'ActionDispatch/DebugExceptions.html',
+      'ActionDispatch/Journey/Parser.html',
+      'ActionDispatch/Reloader.html',
+      'ActionDispatch/Routing/HtmlTableFormatter.html',
+      'ActionDispatch/ShowExceptions.html',
+      'ActionView/FixtureResolver.html',
+      'ActionView/LogSubscriber.html',
+      'ActionView/TestCase/Behavior/RenderedViewsCollection.html',
+      'ActiveRecord/Tasks/DatabaseTasks.html',
+      'ActiveSupport/Dependencies/WatchStack.html',
+      'ActiveSupport/Notifications/Fanout.html',
+      'ActionDispatch/www.example.com',
+      'ActionDispatch/Http/www.rubyonrails.org',
+      'ActionDispatch/Http/www.rubyonrails.co.uk',
+      '\'TZ\'',
+      'active_record_migrations.html',
+      'association_basics.html'
+    ]
+
+    options[:skip_patterns] = [
+      /history/i,
+      /rakefile/i,
+      /changelog/i,
+      /readme/i,
+      /news/i,
+      /license/i,
+      /release_notes/,
+      /\AActionController\/Testing/,
+      /\AActionView\/LookupContext/,
+      /\AActionView\/Resolver/,
+      /\AActiveSupport\/Multibyte\/Unicode\//,
+      /\AActiveSupport\/XML/i,
+      /\ASourceAnnotationExtractor/,
+      /\AI18n\/Railtie/,
+      /\AMinitest/,
+      /\ARails\/API/,
+      /\ARails\/AppBuilder/,
+      /\ARails\/PluginBuilder/,
+      /\ARails\/Generators\/Testing/
+    ]
+
+    options[:attribution] = ->(filter) do
+      if filter.slug.start_with?('guides')
+        <<-HTML
+          &copy; 2004&ndash;2021 David Heinemeier Hansson<br>
+          Licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
+        HTML
+      else
+        <<-HTML
+          &copy; 2004&ndash;2021 David Heinemeier Hansson<br>
+          Licensed under the MIT License.
+        HTML
+      end
+    end
+
+    version '6.1' do
+      self.release = '6.1.3.2'
+
+      self.base_urls = [
+      'https://api.rubyonrails.org/',
+      'https://guides.rubyonrails.org/'
+      ]
+
+      options[:skip_patterns] << /v.*\..*\//
+    end
+
+    version '6.0' do
+      self.release = '6.0.0'
+    end
+
+    version '5.2' do
+      self.release = '5.2.2'
+    end
+
+    version '5.1' do
+      self.release = '5.1.6'
+    end
+
+    version '5.0' do
+      self.release = '5.0.7'
+    end
+
+    version '4.2' do
+      self.release = '4.2.11'
+    end
+
+    version '4.1' do
+      self.release = '4.1.16'
+    end
+
+    def get_latest_version(opts)
+      doc = fetch_doc('https://rubyonrails.org/', opts)
+      doc.at_css('.version p a').content.scan(/\d\.\d*\.*\d*\.*\d*/)[0]
+    end
+
+  end
+end

lib/docs/scrapers/rdoc/rails.rb

@@ -1,119 +0,0 @@
-module Docs
-  class Rails < Rdoc
-    # Instructions:
-    #   1. Download a release at https://github.com/rails/rails/releases
-    #   2. Open "railties/lib/rails/api/task.rb" and comment out any code related to sdoc ("configure_sdoc")
-    #   3. Run "bundle install --without db && bundle exec rake rdoc" (in the Rails directory)
-    #   4. Run "cd guides && bundle exec rake guides:generate:html"
-    #   5. Copy the "guides/output" directory to "html/guides"
-    #   6. Copy the "html" directory to "docs/rails~[version]"
-
-    include FixInternalUrlsBehavior
-
-    self.name = 'Ruby on Rails'
-    self.slug = 'rails'
-    self.initial_paths = %w(guides/index.html)
-    self.links = {
-      home: 'http://rubyonrails.org/',
-      code: 'https://github.com/rails/rails'
-    }
-
-    html_filters.replace 'rdoc/entries', 'rails/entries'
-    html_filters.push 'rails/clean_html_guides'
-
-    options[:skip_rdoc_filters?] = ->(filter) { filter.slug.start_with?('guides/') }
-
-    options[:root_title] = 'Ruby on Rails'
-
-    options[:skip] += %w(
-      guides/credits.html
-      guides/ruby_on_rails_guides_guidelines.html
-      guides/contributing_to_ruby_on_rails.html
-      guides/development_dependencies_install.html
-      guides/api_documentation_guidelines.html
-      ActionController/Instrumentation.html
-      ActionController/Rendering.html
-      ActionDispatch/DebugExceptions.html
-      ActionDispatch/Journey/Parser.html
-      ActionDispatch/Reloader.html
-      ActionDispatch/Routing/HtmlTableFormatter.html
-      ActionDispatch/ShowExceptions.html
-      ActionView/FixtureResolver.html
-      ActionView/LogSubscriber.html
-      ActionView/TestCase/Behavior/RenderedViewsCollection.html
-      ActiveRecord/Tasks/DatabaseTasks.html
-      ActiveSupport/Dependencies/WatchStack.html
-      ActiveSupport/Notifications/Fanout.html)
-
-    # False positives found by docs:generate
-    options[:skip].concat %w(
-      ActionDispatch/www.example.com
-      ActionDispatch/Http/www.rubyonrails.org
-      ActionDispatch/Http/www.rubyonrails.co.uk
-      'TZ'
-      active_record_migrations.html
-      association_basics.html)
-
-    options[:skip_patterns] += [
-      /release_notes/,
-      /\AActionController\/Testing/,
-      /\AActionView\/LookupContext/,
-      /\AActionView\/Resolver/,
-      /\AActiveSupport\/Multibyte\/Unicode\//,
-      /\AActiveSupport\/XML/i,
-      /\ASourceAnnotationExtractor/,
-      /\AI18n\/Railtie/,
-      /\AMinitest/,
-      /\ARails\/API/,
-      /\ARails\/AppBuilder/,
-      /\ARails\/PluginBuilder/,
-      /\ARails\/Generators\/Testing/]
-
-    options[:attribution] = ->(filter) do
-      if filter.slug.start_with?('guides')
-        <<-HTML
-          &copy; 2004&ndash;2020 David Heinemeier Hansson<br>
-          Licensed under the Creative Commons Attribution-ShareAlike 4.0 International License.
-        HTML
-      else
-        <<-HTML
-          &copy; 2004&ndash;2020 David Heinemeier Hansson<br>
-          Licensed under the MIT License.
-        HTML
-      end
-    end
-
-    version '6.1' do
-      self.release = '6.1.1'
-    end
-
-    version '6.0' do
-      self.release = '6.0.0'
-    end
-
-    version '5.2' do
-      self.release = '5.2.2'
-    end
-
-    version '5.1' do
-      self.release = '5.1.6'
-    end
-
-    version '5.0' do
-      self.release = '5.0.7'
-    end
-
-    version '4.2' do
-      self.release = '4.2.11'
-    end
-
-    version '4.1' do
-      self.release = '4.1.16'
-    end
-
-    def get_latest_version(opts)
-      doc = fetch_doc('https://rubyonrails.org/', opts)
-      doc.at_css('.version p a').content.scan(/\d\.\d*\.*\d*\.*\d*/)[0]
-    end
-  end
-end