Add SQLite documentation

assets/javascripts/news.json

@@ -1,5 +1,8 @@
 [
   [
+    "2016-12-04",
+    "New documentation: <a href=\"/sqlite/\">SQLite</a>"
+  ], [
     "2016-11-20",
     "New documentations: <a href=\"/yarn/\">Yarn</a>, <a href=\"/immutable/\">Immutable.js</a> and <a href=\"/async/\">Async</a>"
   ], [

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -529,6 +529,11 @@ credits = [
     '2014-2015 Automattic',
     'MIT',
     'https://raw.githubusercontent.com/Automattic/socket.io/master/LICENSE'
+  ], [
+    'SQLite',
+    'n/a',
+    'Public Domain',
+    'https://sqlite.org/copyright.html'
   ], [
     'Statsmodels',
     '2009-2012 Statsmodels Developers<br>&copy; 2006-2008 Scipy Developers<br>&copy; 2006 Jonathan E. Taylor',

assets/javascripts/views/pages/sqlite.coffee

@@ -0,0 +1,17 @@
+#= require views/pages/simple
+
+class app.views.SqlitePage extends app.views.SimplePage
+  @events:
+    click: 'onClick'
+
+  onClick: (event) =>
+    return unless id = event.target.getAttribute('data-toggle')
+    return unless el = @find("##{id}")
+    $.stopEvent(event)
+    if el.style.display == 'none'
+      el.style.display = 'block'
+      event.target.textContent = 'hide'
+    else
+      el.style.display = 'none'
+      event.target.textContent = 'show'
+    return

assets/stylesheets/application-dark.css.scss

@@ -84,6 +84,7 @@
         'pages/socketio',
         'pages/sphinx',
         'pages/sphinx_simple',
+        'pages/sqlite',
         'pages/support_tables',
         'pages/tcl_tk',
         'pages/tensorflow',

assets/stylesheets/application.css.scss

@@ -84,6 +84,7 @@
         'pages/socketio',
         'pages/sphinx',
         'pages/sphinx_simple',
+        'pages/sqlite',
         'pages/support_tables',
         'pages/tcl_tk',
         'pages/tensorflow',

assets/stylesheets/components/_content.scss

@@ -391,6 +391,19 @@
   &._pre-clip-error:after { content: 'Error'; }
 }
 
+._btn {
+  white-space: nowrap;
+  padding: .125rem .375rem;
+  background-image: linear-gradient(lighten($boxBackground, 3%), darken($boxBackground, 4%));
+  border: 1px solid $boxBorder;
+  border-radius: 3px;
+
+  &:active {
+    background-color: $boxBackground;
+    box-shadow: inset 0 1px 3px rgba(black, .15);
+  }
+}
+
 ._github-btn {
   display: inline-block;
   vertical-align: text-top;

assets/stylesheets/global/_icons.scss

@@ -56,6 +56,7 @@
 %icon-close-white           { background-position: -2rem -5rem; }
 %icon-back                  { background-position: -3rem -5rem; @extend %darkIconFix !optional; }
 
+._icon-sqlite:before        { background-position: -5rem 0; @extend %darkIconFix !optional; }
 ._icon-async:before         { background-position: -6rem 0; @extend %darkIconFix !optional; }
 ._icon-http:before          { background-position: -7rem 0; @extend %darkIconFix !optional; }
 ._icon-jquery:before        { background-position: -8rem 0; @extend %darkIconFix !optional; }

assets/stylesheets/pages/_sqlite.scss

@@ -0,0 +1,6 @@
+._sqlite {
+  @extend %simple;
+
+  dt { @extend %block-label, %label-blue; }
+  .todo { @extend %note, %note-red; }
+}

lib/docs/core/parser.rb

@@ -1,11 +1,10 @@
 module Docs
   class Parser
+    attr_reader :title, :html
+
     def initialize(content)
       @content = content
-    end
+      @html = document? ? parse_as_document : parse_as_fragment
-
-    def html
-      @html ||= document? ? parse_as_document : parse_as_fragment
     end
 
     private
@@ -16,6 +15,7 @@ module Docs
 
     def parse_as_document
       document = Nokogiri::HTML.parse @content, nil, 'UTF-8'
+      @title = document.at_css('title').try(:content)
       document.at_css 'body'
     end
 

lib/docs/core/scraper.rb

@@ -171,7 +171,10 @@ module Docs
 
     def process_response(response)
       data = {}
-      pipeline.call(parse(response.body), pipeline_context(response), data)
+      html, title = parse(response.body)
+      context = pipeline_context(response)
+      context[:html_title] = title
+      pipeline.call(html, context, data)
       data
     end
 
@@ -180,7 +183,8 @@ module Docs
     end
 
     def parse(string)
-      Parser.new(string).html
+      parser = Parser.new(string)
+      [parser.html, parser.title]
     end
 
     def with_filters(*filters)

lib/docs/filters/core/clean_html.rb

@@ -1,7 +1,7 @@
 module Docs
   class CleanHtmlFilter < Filter
     def call
-      css('script', 'style').remove
+      css('script', 'style', 'link').remove
       xpath('descendant::comment()').remove
       xpath('./text()', './/text()[not(ancestor::pre) and not(ancestor::code) and not(ancestor::div[contains(concat(" ", normalize-space(@class), " "), " prism ")])]').each do |node|
         content = node.content

lib/docs/filters/git/entries.rb

@@ -15,7 +15,7 @@ module Docs
         elsif slug == 'git' || slug.start_with?('git-')
           'Git'
         else
-          'Miscellaenous'
+          'Miscellaneous'
         end
       end
     end

lib/docs/filters/sqlite/clean_html.rb

@@ -0,0 +1,92 @@
+module Docs
+  class Sqlite
+    class CleanHtmlFilter < Filter
+      def call
+        at_css('.nosearch').remove
+
+        css('.rightsidebar', 'hr', '.sh_mark', '.fancy_toc > a', '.fancy_toc_mark', 'h[style*="none"]',
+            'a[href$="intro.html"] > h2', 'a[href$="intro"] > h2', '#document_title + #toc_header',
+            '#document_title ~ #toc').remove
+
+        css('.fancy_title', '> h2[align=center]', '#document_title').each do |node|
+          node.name = 'h1'
+        end
+
+        unless at_css('h1')
+          if at_css('h2').content == context[:html_title]
+            at_css('h2').name = 'h1'
+          else
+            doc.child.before("<h1>#{context[:html_title]}</h1>")
+          end
+        end
+
+        if root_page?
+          at_css('h1').content = 'SQLite Documentation'
+        end
+
+        css('.codeblock', '.fancy', '.nosearch', '.optab > blockquote', '.optab').each do |node|
+          node.before(node.children).remove
+        end
+
+        css('blockquote').each do |node|
+          next unless node.at_css('b')
+          node.name = 'pre'
+          node.inner_html = node.inner_html.gsub('<br>', "\n")
+        end
+
+        css('.todo').each do |node|
+          node.inner_html = "TODO: #{node.inner_html}"
+        end
+
+        css('blockquote > pre', 'a > h1', 'a > h2', 'center > table', 'ul > ul').each do |node|
+          node.parent.before(node.parent.children).remove
+        end
+
+        css('table > tr:first-child:last-child > td:first-child:last-child > pre',
+            'table > tr:first-child:last-child > td:first-child:last-child > ul').each do |node|
+          node.ancestors('table').first.replace(node)
+        end
+
+        css('a[name]').each do |node|
+          if node.next_element
+            if node.next_element['id']
+              node.next_element.next_element['id'] ||= node['name']
+            else
+              node.next_element['id'] = node['name']
+            end
+          else
+            node.parent['id'] ||= node['name']
+          end
+          node.remove
+        end
+
+        unless at_css('h2')
+          css('h1 ~ h1').each do |node|
+            node.name = 'h2'
+          end
+        end
+
+        css('tt').each do |node|
+          node.name = 'code'
+        end
+
+        css('pre').each do |node|
+          node.content = node.content
+          node['data-language'] = 'sql'
+        end
+
+        css('button[onclick]').each do |node|
+          node['class'] = '_btn'
+          node['data-toggle'] = node['onclick'][/hideorshow\("\w+","(\w+)"\)/, 1]
+          node.remove_attribute('onclick')
+        end
+
+        css('*[align]').remove_attr('align')
+        css('*[style]:not(.imgcontainer)').remove_attr('style')
+        css('table[border]').remove_attr('border')
+
+        doc
+      end
+    end
+  end
+end

lib/docs/filters/sqlite/clean_js_tables.rb

@@ -0,0 +1,42 @@
+module Docs
+  class Sqlite
+    class CleanJsTablesFilter < Filter
+      def call
+        css('table[id]:empty + script').each do |node|
+          json_list = JSON.parse(node.inner_html[/\[.+?\]/m])
+          list = '<ul>'
+          json_list.each do |item|
+            list << '<li>'
+
+            unless item['u'].blank? || item['s'] == 2
+              list << %(<a href="#{item['u']}">)
+              link = true
+            end
+
+            if item['s'] == 2 || item['s'] == 3
+              list << "<s>#{item['x']}</s>"
+            else
+              list << item['x']
+            end
+
+            list << '</a>' if link
+
+            if item['s'] == 1
+              list << ' <small><i>(exp)</i></small>'
+            elsif item['s'] == 3
+              list << '&sup1;'
+            elsif item['s'] == 4
+              list << '&sup2;'
+            elsif item['s'] == 5
+              list << '&sup3;'
+            end
+          end
+          list << '</ul>'
+          node.previous_element.replace(list)
+        end
+
+        doc
+      end
+    end
+  end
+end

lib/docs/filters/sqlite/entries.rb

@@ -0,0 +1,79 @@
+module Docs
+  class Sqlite
+    class EntriesFilter < Docs::EntriesFilter
+      ADDITIONAL_ENTRIES = {}
+
+      def get_name
+        name = context[:html_title]
+        name.remove! 'SQLite Query Language: '
+        name.remove! %r{\.\z}
+        name = at_css('#document_title').content if name == 'No Title'
+        name
+      end
+
+      TYPE_BY_SUBPATH_STARTS_WITH = {
+        'c3ref' => 'C Interface',
+        'capi' => 'C Interface',
+        'session' => 'C Interface: Session Module',
+        'optoverview' => 'Query Planner',
+        'queryplanner' => 'Query Planner',
+        'syntax' => 'Syntax Diagrams',
+        'lang' => 'Language',
+        'pragma' => 'PRAGMA Statements',
+        'cli' => 'CLI',
+        'json' => 'JSON',
+        'fileformat' => 'Database File Format',
+        'tcl' => 'Tcl Interface',
+        'malloc' => 'Dynamic Memory Allocation',
+        'vtab' => 'Virtual Table Mechanism',
+        'datatype' => 'Datatypes',
+        'locking' => 'Locking and Concurrency',
+        'foreignkey' => 'Foreign Key Constraints',
+        'wal' => 'Write-Ahead Logging',
+        'fts' => 'Full-Text Search',
+        'rtree' => 'R*Tree Module',
+        'rbu' => 'RBU Extension',
+        'limits' => 'Limits',
+        'howtocorrupt' => 'How To Corrupt'
+      }
+
+      def get_type
+        TYPE_BY_SUBPATH_STARTS_WITH.each_pair do |key, value|
+          return value if subpath.start_with?(key)
+        end
+
+        if slug.in?(%w(cintro carray c_interface))
+          'C Interface'
+        elsif context[:html_title].start_with?('SQLite Query Language')
+          'Query Language'
+        else
+          'Miscellaneous'
+        end
+      end
+
+      IGNORE_ADDITIONAL_ENTRIES_SLUGS = %w(testing uri getthecode whentouse compile)
+
+      def additional_entries
+        if subpath == 'keyword_index.html'
+          css('li a[href*="#"]').each do |node|
+            slug, id = node['href'].split('#')
+            name = node.content.strip
+            next if name.start_with?('-') || IGNORE_ADDITIONAL_ENTRIES_SLUGS.include?(slug)
+            name.remove! %r{\Athe }
+            name.remove! %r{ SQL function\z}
+            name = 'ORDER BY' if name == 'order by'
+            name.sub!(/\A([a-z])([a-z]+) /) { "#{$1.upcase}#{$2} " }
+            ADDITIONAL_ENTRIES[slug] ||= []
+            ADDITIONAL_ENTRIES[slug] << [name, id]
+          end
+        end
+
+        ADDITIONAL_ENTRIES[slug] || []
+      end
+
+      def include_default_entry?
+        subpath != 'keyword_index.html' && subpath != 'sitemap.html'
+      end
+    end
+  end
+end

lib/docs/scrapers/sqlite.rb

@@ -0,0 +1,52 @@
+module Docs
+  class Sqlite < FileScraper
+    self.name = 'SQLite'
+    self.type = 'sqlite'
+    self.release = '3.15.2'
+    self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/'
+    self.base_url = 'https://sqlite.org/'
+    self.root_path = 'docs.html'
+    self.initial_paths = %w(keyword_index.html)
+    self.links = {
+      home: 'https://sqlite.org/',
+      code: 'https://www.sqlite.org/src/'
+    }
+
+    html_filters.insert_before 'clean_html', 'sqlite/clean_js_tables'
+    html_filters.push 'sqlite/entries', 'sqlite/clean_html'
+
+    options[:only_patterns] = [/\.html\z/]
+    options[:skip_patterns] = [/releaselog/, /consortium/]
+    options[:skip] = %w(
+      index.html
+      about.html
+      download.html
+      copyright.html
+      support.html
+      prosupport.html
+      hp1.html
+      news.html
+      oldnews.html
+      doclist.html
+      dev.html
+      chronology.html
+      not-found.html
+      famous.html
+      books.html
+      crew.html
+      mostdeployed.html
+      requirements.html
+      session/intro.html
+      syntax.html
+    )
+
+    options[:attribution] = 'SQLite is in the Public Domain.'
+
+    private
+
+    def parse(html)
+      html.gsub! %r{(<h2[^>]*>[^<]+)</h1>}, '\1</h2>'
+      super
+    end
+  end
+end

test/lib/docs/core/parser_test.rb

@@ -28,4 +28,16 @@ class DocsParserTest < MiniTest::Spec
       end
     end
   end
+
+  describe "#title" do
+    it "returns nil when there is no <title>" do
+      body = '<!doctype html><meta charset=utf-8><div>Test</div>'
+      assert_nil parser(body).title
+    end
+
+    it "returns the <title> when there is one" do
+      body = '<!doctype html><meta charset=utf-8><title>Title</title><div>Test</div>'
+      assert_equal 'Title', parser(body).title
+    end
+  end
 end