Improve PHPUnit scraper

assets/stylesheets/pages/_phpunit.scss

@@ -1,21 +1,18 @@
 ._phpunit {
-  h1 {
+  > h2 { @extend %block-heading; }
-    margin-top: 0;
+  > h3 { @extend %block-label, %label-blue; }
-    @extend %lined-heading;
+  > h4 { font-size: 1em; }
-  }
 
-  h2.title {
+  > p > code { @extend %label; }
-    @extend %block-heading;
-  }
 
-  .programlisting > pre { white-space: normal; }
+  .warning, .alert {
+    @extend %note;
 
-  .literal {
+    > h3 {
-    padding: 2px 4px;
+      margin: 0 0 .5em;
-    color: #c7254e;
+      font-size: 1em;
-    background-color: #f9f2f4;
+    }
-    border-radius: 4px;
   }
 
-  div.warning { @extend %note, %note-red; }
+  .alert-danger { @extend %note-red; }
 }

lib/docs/filters/phpunit/clean_html.rb

@@ -7,58 +7,52 @@ module Docs
       end
 
       def root
-        doc.inner_html = <<-HTML
+        doc.inner_html = ' '
-          <p>PHPUnit is a programmer-oriented testing framework for PHP.<br>
-          It is an instance of the xUnit architecture for unit testing frameworks.</p>
-        HTML
       end
 
       def other
-        # set root on appendix
+        @doc = doc.at_css('div.appendix, div.chapter')
-        @doc = doc.at_css('div.appendix')
 
-        # remove attributes 'style'
+        css('.example-break', '.table-break').remove
-        css('*').remove_attr('style')
 
-        # clean titles
+        css('a[id]').each do |node|
-        css('div.titlepage').each do |node|
+          next unless node.content.blank?
-          title = node.at_css('.title')
+          node.parent['id'] = node['id']
-          case title.name
+          node.remove
-          when 'h1'
-            # remove 'Appendix X.' from top title
-            nodetitle = title.content
-            title.content = nodetitle.gsub(/Appendix \w+\. /, '')
-          when 'h2'
-            # set link anchors in entries (title level 2)
-            anchor = Nokogiri::XML::Node.new "a", @doc
-            anchor.content = title.content
-            anchor['id'] = title.content.downcase.gsub(/[^a-z]/, '')
-            title.content = ''
-            anchor.parent = title
-          end
-          node.replace title
         end
 
-        # set anchor for internal references
+        css('.titlepage').each do |node|
-        css('p.title').each do |node|
+          title = node.at_css('h1, .title')
-          anchor = Nokogiri::XML::Node.new "a", @doc
+          title.content = title.content.remove(/(Chapter|Appendix)\s+\w+\.\s+/)
-          anchor.content = node.content
+          node.before(title).remove
-          anchor['id'] = anchor.content[/\w+ [A-z0-9.]+/].downcase.parameterize
-          node.content = ''
-          anchor.parent = node
         end
 
-        # clean internal references
+        css('.section').each do |node|
-        css('a').each do |node|
+          node.before(node.children).remove
-          page = node['href'][/([A-z.-]+)?#/, 1] if node['href']
-          if page then
-            page  = page + '.html' unless page[/.*\.html/]
-            if Phpunit.initial_paths.include? page
-              node['href'] = node['href'].gsub(/#[A-z.-]+/, '#' + node.content.downcase.parameterize)
-            end
-          end
         end
 
+        css('[style], [border], [valign]').each do |node|
+          node.remove_attribute('style')
+          node.remove_attribute('border')
+          node.remove_attribute('valign')
+        end
+
+        css('.warning h3', '.alert h3').each do |node|
+          node.remove if node.content == 'Note'
+        end
+
+        css('p > code.literal:first-child:last-child').each do |node|
+          next if node.previous_sibling && node.previous_sibling.content.present?
+          next if node.next_sibling && node.next_sibling.content.present?
+          node.parent.name = 'pre'
+          node.parent.content = node.content
+        end
+
+        css('pre', '.term').each do |node|
+          node.content = node.content
+        end
+
+        doc
       end
     end
   end

lib/docs/filters/phpunit/entries.rb

@@ -1,19 +1,24 @@
 module Docs
   class Phpunit
     class EntriesFilter < Docs::EntriesFilter
-      def additional_entries
+      def get_name
-        entries = []
+        at_css('h1').content
+      end
+
+      def get_type
+        if name.in?(%w(Assertions Annotations))
+          name
+        else
+          'Guides'
+        end
+      end
 
-        if at_css('h1')
+      def additional_entries
-          type = at_css('h1').content.gsub(/Appendix \w+\. /, '')
+        return [] if type == 'Guides'
 
-          css('h2').each do |node|
+        css('h2').map do |node|
-            name = node.content
+          [node.content, node['id']]
-            id = name.parameterize
-            entries << [name, id, type]
-          end
         end
-        entries
       end
     end
   end

lib/docs/scrapers/phpunit.rb

@@ -2,26 +2,19 @@ module Docs
   class Phpunit < UrlScraper
     self.name = 'PHPUnit'
     self.type = 'phpunit'
-    self.slug = 'phpunit'
     self.version = '4.3'
-    self.base_url = 'https://phpunit.de/manual/4.3/en/'
+    self.base_url = "https://phpunit.de/manual/#{version}/en/"
-    self.initial_paths = %w(appendixes.assertions.html appendixes.annotations.html)
+    self.root_path = 'index.html'
 
-    html_filters.push 'phpunit/entries', 'phpunit/clean_html', 'title'
+    html_filters.push 'phpunit/clean_html', 'phpunit/entries', 'title'
-
-    options[:skip_links] = true
 
+    options[:root_title] = 'PHPUnit'
     options[:title] = false
-    options[:root_title] = "#{self.name} #{self.version}"
-
-    options[:fix_urls] = ->(url) do
-      if self.initial_paths.include? url[/\/([A-z.-]+)#/, 1]
-        url = url[/#(.+)/, 1].downcase
-        url.gsub! /(\w+\.\w+)\.(\w+)/, '\1#\2'
-      end
-      url
-    end
 
+    options[:skip] = %w(
+      appendixes.index.html
+      appendixes.bibliography.html
+      appendixes.copyright.html)
 
     options[:attribution] = <<-HTML
       &copy; 2005&ndash;2014 Sebastian Bergmann<br>