Finish Twig scraper

assets/javascripts/news.json

@@ -1,5 +1,8 @@
 [
   [
+    "2016-09-18",
+    "New documentation: <a href=\"/twig/\">Twig</a>"
+  ], [
     "2016-09-05",
     "New documentations: <a href=\"/fish/\">Fish</a>, <a href=\"/bottle/\">Bottle</a> and <a href=\"/scikit_image/\">scikit-image</a>"
   ], [

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -524,6 +524,11 @@ credits = [
     '2015 The TensorFlow Authors',
     'Apache',
     'https://raw.githubusercontent.com/tensorflow/tensorflow/master/LICENSE'
+  ], [
+    'Twig',
+    '2009-2016 The Twig Team',
+    'BSD',
+    'http://twig.sensiolabs.org/license'
   ], [
     'TypeScript',
     'Microsoft and other contributors',

assets/stylesheets/application-dark.css.scss

@@ -84,8 +84,8 @@
         'pages/socketio',
         'pages/sphinx',
         'pages/sphinx_simple',
-        'pages/tcl_tk',
         'pages/support_tables',
+        'pages/tcl_tk',
         'pages/tensorflow',
         'pages/underscore',
         'pages/vagrant',

assets/stylesheets/global/_icons.scss

@@ -118,6 +118,7 @@
 ._icon-bower:before         { background-position: -4rem -6rem; }
 ._icon-fish:before          { background-position: -5rem -6rem; @extend %darkIconFix !optional; }
 ._icon-scikit_image:before  { background-position: -6rem -6rem; }
+._icon-twig:before          { background-position: -7rem -6rem; }
 ._icon-bottle:before        { background-position: 0 -7rem; }
 ._icon-docker:before        { background-position: -1rem -7rem; }
 ._icon-cakephp:before       { background-position: -2rem -7rem; }

lib/docs/filters/twig/clean_html.rb

@@ -2,8 +2,36 @@ module Docs
   class Twig
     class CleanHtmlFilter < Filter
       def call
+        css('.infobar', '.offline-docs', '.headerlink').remove
 
-        css('.infobar', '.admonition-note', 'ul.pages', '.offline-docs').remove
+        css('.builtin-reference', '.admonition-wrapper', 'h1 > code', 'h2 > code', '.body-web', '.reference em').each do |node|
+          node.before(node.children).remove
+        end
+
+        css('.section').each do |node|
+          node.first_element_child['id'] = node['id'] if node['id']
+          node.before(node.children).remove
+        end
+
+        doc.child.remove until doc.child.name == 'h1'
+
+        css('.literal-block').each do |node|
+          pre = node.at_css('.highlight pre') || node.at_css('pre')
+          pre.content = pre.content
+          node.replace(pre)
+        end
+
+        css('p.versionadded').each do |node|
+          node.name = 'div'
+        end
+
+        css('h3:contains("Arguments")').each do |node|
+          node.name = 'h4'
+        end
+
+        css('.navigation').each do |node|
+          node['style'] = 'text-align: center'
+        end
 
         doc
       end

lib/docs/filters/twig/entries.rb

@@ -2,32 +2,24 @@ module Docs
   class Twig
     class EntriesFilter < Docs::EntriesFilter
       def get_name
-        name = at_css('h1').content.strip.tr('¶', '')
+        at_css('h1').content
-        name
       end
 
       def get_type
-        if slug.include?('deprecate')
+        if slug.start_with?('filters')
-          'Deprecated'
+          'Filters'
-        elsif slug.include?('extensions')
+        elsif slug.start_with?('functions')
-          'Doc\\Extensions'
+          'Functions'
-        elsif slug.include?('tags')
+        elsif slug.start_with?('tags')
-          'tags'
+          'Tags'
-        elsif slug.include?('filters')
+        elsif slug.start_with?('tests')
-          'filters'
+          'Tests'
-        elsif slug.include?('functions')
+        elsif slug.start_with?('extensions')
-          'functions'
+          'Extensions'
-        elsif slug.include?('tests')
+        else
-          'tests'
+          'Miscellaneous'
-        elsif slug.include?('-operator') || slug.include?('#math') || slug.include?('comparisons')
-          'operators'
-        elsif slug.in?('doc/index') || slug.include?('intro') || slug.include?('recipes') || slug.include?('internals') || slug.include?('coding_standards') || slug.include?('installation') ||  slug.include?('api') || slug.include?('advanced')
-          'Doc'
-        elsif slug.include?('templates')
-          'Doc\\Templates'
         end
       end
-
     end
   end
 end

lib/docs/scrapers/twig.rb

@@ -1,20 +1,20 @@
 module Docs
   class Twig < UrlScraper
-    self.type = 'twig'
+    self.type = 'sphinx'
+    self.release = '1.24'
+    self.base_url = 'http://twig.sensiolabs.org/doc/'
+    self.root_path = 'index.html'
+    self.initial_paths = %w(extensions/index.html)
 
     options[:attribution] = <<-HTML
-      &copy; 2009&ndash;2016 by SensioLabs<br>
+      &copy; 2009&ndash;2016 by the Twig Team<br>
-      Licensed under the three clause BSD license.
+      Licensed under the three clause BSD license.<br>
+      The Twig logo is &copy; 2010&ndash;2016 SensioLabs
     HTML
 
-    self.release = '1.24.1'
-    self.base_url = 'http://twig.sensiolabs.org/'
-    self.root_path = 'documentation'
-
     html_filters.push 'twig/clean_html', 'twig/entries'
 
     options[:container] = 'div.bd > div.content'
-    options[:skip_patterns] = [/\Aapi/, /\Alicense/]
+    options[:skip] = %w(deprecated.html advanced_legacy.html)
-    options[:skip] = %w(doc/deprecated.html doc/advanced_legacy.html)
   end
 end