diff --git a/assets/images/docs.png b/assets/images/docs.png index e5fcbd0d..826d4faa 100644 Binary files a/assets/images/docs.png and b/assets/images/docs.png differ diff --git a/assets/images/docs@2x.png b/assets/images/docs@2x.png index 285a61eb..ca6b4931 100644 Binary files a/assets/images/docs@2x.png and b/assets/images/docs@2x.png differ diff --git a/assets/javascripts/news.json b/assets/javascripts/news.json index 3ac79b10..92933f02 100644 --- a/assets/javascripts/news.json +++ b/assets/javascripts/news.json @@ -1,5 +1,8 @@ [ [ + "2016-12-04", + "New documentation: SQLite" + ], [ "2016-11-20", "New documentations: Yarn, Immutable.js and Async" ], [ diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee index 19a3dd2e..2cab5f0c 100644 --- a/assets/javascripts/templates/pages/about_tmpl.coffee +++ b/assets/javascripts/templates/pages/about_tmpl.coffee @@ -529,6 +529,11 @@ credits = [ '2014-2015 Automattic', 'MIT', 'https://raw.githubusercontent.com/Automattic/socket.io/master/LICENSE' + ], [ + 'SQLite', + 'n/a', + 'Public Domain', + 'https://sqlite.org/copyright.html' ], [ 'Statsmodels', '2009-2012 Statsmodels Developers
© 2006-2008 Scipy Developers
© 2006 Jonathan E. Taylor', diff --git a/assets/javascripts/views/pages/sqlite.coffee b/assets/javascripts/views/pages/sqlite.coffee new file mode 100644 index 00000000..a6975cb3 --- /dev/null +++ b/assets/javascripts/views/pages/sqlite.coffee @@ -0,0 +1,17 @@ +#= require views/pages/simple + +class app.views.SqlitePage extends app.views.SimplePage + @events: + click: 'onClick' + + onClick: (event) => + return unless id = event.target.getAttribute('data-toggle') + return unless el = @find("##{id}") + $.stopEvent(event) + if el.style.display == 'none' + el.style.display = 'block' + event.target.textContent = 'hide' + else + el.style.display = 'none' + event.target.textContent = 'show' + return diff --git a/assets/stylesheets/application-dark.css.scss b/assets/stylesheets/application-dark.css.scss index 407d1bc0..5846183b 100644 --- a/assets/stylesheets/application-dark.css.scss +++ b/assets/stylesheets/application-dark.css.scss @@ -84,6 +84,7 @@ 'pages/socketio', 'pages/sphinx', 'pages/sphinx_simple', + 'pages/sqlite', 'pages/support_tables', 'pages/tcl_tk', 'pages/tensorflow', diff --git a/assets/stylesheets/application.css.scss b/assets/stylesheets/application.css.scss index 1d7798f1..ac52f946 100644 --- a/assets/stylesheets/application.css.scss +++ b/assets/stylesheets/application.css.scss @@ -84,6 +84,7 @@ 'pages/socketio', 'pages/sphinx', 'pages/sphinx_simple', + 'pages/sqlite', 'pages/support_tables', 'pages/tcl_tk', 'pages/tensorflow', diff --git a/assets/stylesheets/components/_content.scss b/assets/stylesheets/components/_content.scss index 647c2914..950debcf 100644 --- a/assets/stylesheets/components/_content.scss +++ b/assets/stylesheets/components/_content.scss @@ -391,6 +391,19 @@ &._pre-clip-error:after { content: 'Error'; } } +._btn { + white-space: nowrap; + padding: .125rem .375rem; + background-image: linear-gradient(lighten($boxBackground, 3%), darken($boxBackground, 4%)); + border: 1px solid $boxBorder; + border-radius: 3px; + + &:active { + background-color: $boxBackground; + box-shadow: inset 0 1px 3px rgba(black, .15); + } +} + ._github-btn { display: inline-block; vertical-align: text-top; diff --git a/assets/stylesheets/global/_icons.scss b/assets/stylesheets/global/_icons.scss index 21df3ac3..a6e8c7df 100644 --- a/assets/stylesheets/global/_icons.scss +++ b/assets/stylesheets/global/_icons.scss @@ -56,6 +56,7 @@ %icon-close-white { background-position: -2rem -5rem; } %icon-back { background-position: -3rem -5rem; @extend %darkIconFix !optional; } +._icon-sqlite:before { background-position: -5rem 0; @extend %darkIconFix !optional; } ._icon-async:before { background-position: -6rem 0; @extend %darkIconFix !optional; } ._icon-http:before { background-position: -7rem 0; @extend %darkIconFix !optional; } ._icon-jquery:before { background-position: -8rem 0; @extend %darkIconFix !optional; } diff --git a/assets/stylesheets/pages/_sqlite.scss b/assets/stylesheets/pages/_sqlite.scss new file mode 100644 index 00000000..98b850af --- /dev/null +++ b/assets/stylesheets/pages/_sqlite.scss @@ -0,0 +1,6 @@ +._sqlite { + @extend %simple; + + dt { @extend %block-label, %label-blue; } + .todo { @extend %note, %note-red; } +} diff --git a/lib/docs/core/parser.rb b/lib/docs/core/parser.rb index b3bf7915..7ab02634 100644 --- a/lib/docs/core/parser.rb +++ b/lib/docs/core/parser.rb @@ -1,11 +1,10 @@ module Docs class Parser + attr_reader :title, :html + def initialize(content) @content = content - end - - def html - @html ||= document? ? parse_as_document : parse_as_fragment + @html = document? ? parse_as_document : parse_as_fragment end private @@ -16,6 +15,7 @@ module Docs def parse_as_document document = Nokogiri::HTML.parse @content, nil, 'UTF-8' + @title = document.at_css('title').try(:content) document.at_css 'body' end diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index 3e96cc70..a126299a 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -171,7 +171,10 @@ module Docs def process_response(response) data = {} - pipeline.call(parse(response.body), pipeline_context(response), data) + html, title = parse(response.body) + context = pipeline_context(response) + context[:html_title] = title + pipeline.call(html, context, data) data end @@ -180,7 +183,8 @@ module Docs end def parse(string) - Parser.new(string).html + parser = Parser.new(string) + [parser.html, parser.title] end def with_filters(*filters) diff --git a/lib/docs/filters/core/clean_html.rb b/lib/docs/filters/core/clean_html.rb index 906b1975..ddb26013 100644 --- a/lib/docs/filters/core/clean_html.rb +++ b/lib/docs/filters/core/clean_html.rb @@ -1,7 +1,7 @@ module Docs class CleanHtmlFilter < Filter def call - css('script', 'style').remove + css('script', 'style', 'link').remove xpath('descendant::comment()').remove xpath('./text()', './/text()[not(ancestor::pre) and not(ancestor::code) and not(ancestor::div[contains(concat(" ", normalize-space(@class), " "), " prism ")])]').each do |node| content = node.content diff --git a/lib/docs/filters/git/entries.rb b/lib/docs/filters/git/entries.rb index 7ab8da79..8bd93a38 100644 --- a/lib/docs/filters/git/entries.rb +++ b/lib/docs/filters/git/entries.rb @@ -15,7 +15,7 @@ module Docs elsif slug == 'git' || slug.start_with?('git-') 'Git' else - 'Miscellaenous' + 'Miscellaneous' end end end diff --git a/lib/docs/filters/sqlite/clean_html.rb b/lib/docs/filters/sqlite/clean_html.rb new file mode 100644 index 00000000..09d152d3 --- /dev/null +++ b/lib/docs/filters/sqlite/clean_html.rb @@ -0,0 +1,92 @@ +module Docs + class Sqlite + class CleanHtmlFilter < Filter + def call + at_css('.nosearch').remove + + css('.rightsidebar', 'hr', '.sh_mark', '.fancy_toc > a', '.fancy_toc_mark', 'h[style*="none"]', + 'a[href$="intro.html"] > h2', 'a[href$="intro"] > h2', '#document_title + #toc_header', + '#document_title ~ #toc').remove + + css('.fancy_title', '> h2[align=center]', '#document_title').each do |node| + node.name = 'h1' + end + + unless at_css('h1') + if at_css('h2').content == context[:html_title] + at_css('h2').name = 'h1' + else + doc.child.before("

#{context[:html_title]}

") + end + end + + if root_page? + at_css('h1').content = 'SQLite Documentation' + end + + css('.codeblock', '.fancy', '.nosearch', '.optab > blockquote', '.optab').each do |node| + node.before(node.children).remove + end + + css('blockquote').each do |node| + next unless node.at_css('b') + node.name = 'pre' + node.inner_html = node.inner_html.gsub('
', "\n") + end + + css('.todo').each do |node| + node.inner_html = "TODO: #{node.inner_html}" + end + + css('blockquote > pre', 'a > h1', 'a > h2', 'center > table', 'ul > ul').each do |node| + node.parent.before(node.parent.children).remove + end + + css('table > tr:first-child:last-child > td:first-child:last-child > pre', + 'table > tr:first-child:last-child > td:first-child:last-child > ul').each do |node| + node.ancestors('table').first.replace(node) + end + + css('a[name]').each do |node| + if node.next_element + if node.next_element['id'] + node.next_element.next_element['id'] ||= node['name'] + else + node.next_element['id'] = node['name'] + end + else + node.parent['id'] ||= node['name'] + end + node.remove + end + + unless at_css('h2') + css('h1 ~ h1').each do |node| + node.name = 'h2' + end + end + + css('tt').each do |node| + node.name = 'code' + end + + css('pre').each do |node| + node.content = node.content + node['data-language'] = 'sql' + end + + css('button[onclick]').each do |node| + node['class'] = '_btn' + node['data-toggle'] = node['onclick'][/hideorshow\("\w+","(\w+)"\)/, 1] + node.remove_attribute('onclick') + end + + css('*[align]').remove_attr('align') + css('*[style]:not(.imgcontainer)').remove_attr('style') + css('table[border]').remove_attr('border') + + doc + end + end + end +end diff --git a/lib/docs/filters/sqlite/clean_js_tables.rb b/lib/docs/filters/sqlite/clean_js_tables.rb new file mode 100644 index 00000000..4bc0ee8e --- /dev/null +++ b/lib/docs/filters/sqlite/clean_js_tables.rb @@ -0,0 +1,42 @@ +module Docs + class Sqlite + class CleanJsTablesFilter < Filter + def call + css('table[id]:empty + script').each do |node| + json_list = JSON.parse(node.inner_html[/\[.+?\]/m]) + list = '' + node.previous_element.replace(list) + end + + doc + end + end + end +end diff --git a/lib/docs/filters/sqlite/entries.rb b/lib/docs/filters/sqlite/entries.rb new file mode 100644 index 00000000..8ba7a42a --- /dev/null +++ b/lib/docs/filters/sqlite/entries.rb @@ -0,0 +1,79 @@ +module Docs + class Sqlite + class EntriesFilter < Docs::EntriesFilter + ADDITIONAL_ENTRIES = {} + + def get_name + name = context[:html_title] + name.remove! 'SQLite Query Language: ' + name.remove! %r{\.\z} + name = at_css('#document_title').content if name == 'No Title' + name + end + + TYPE_BY_SUBPATH_STARTS_WITH = { + 'c3ref' => 'C Interface', + 'capi' => 'C Interface', + 'session' => 'C Interface: Session Module', + 'optoverview' => 'Query Planner', + 'queryplanner' => 'Query Planner', + 'syntax' => 'Syntax Diagrams', + 'lang' => 'Language', + 'pragma' => 'PRAGMA Statements', + 'cli' => 'CLI', + 'json' => 'JSON', + 'fileformat' => 'Database File Format', + 'tcl' => 'Tcl Interface', + 'malloc' => 'Dynamic Memory Allocation', + 'vtab' => 'Virtual Table Mechanism', + 'datatype' => 'Datatypes', + 'locking' => 'Locking and Concurrency', + 'foreignkey' => 'Foreign Key Constraints', + 'wal' => 'Write-Ahead Logging', + 'fts' => 'Full-Text Search', + 'rtree' => 'R*Tree Module', + 'rbu' => 'RBU Extension', + 'limits' => 'Limits', + 'howtocorrupt' => 'How To Corrupt' + } + + def get_type + TYPE_BY_SUBPATH_STARTS_WITH.each_pair do |key, value| + return value if subpath.start_with?(key) + end + + if slug.in?(%w(cintro carray c_interface)) + 'C Interface' + elsif context[:html_title].start_with?('SQLite Query Language') + 'Query Language' + else + 'Miscellaneous' + end + end + + IGNORE_ADDITIONAL_ENTRIES_SLUGS = %w(testing uri getthecode whentouse compile) + + def additional_entries + if subpath == 'keyword_index.html' + css('li a[href*="#"]').each do |node| + slug, id = node['href'].split('#') + name = node.content.strip + next if name.start_with?('-') || IGNORE_ADDITIONAL_ENTRIES_SLUGS.include?(slug) + name.remove! %r{\Athe } + name.remove! %r{ SQL function\z} + name = 'ORDER BY' if name == 'order by' + name.sub!(/\A([a-z])([a-z]+) /) { "#{$1.upcase}#{$2} " } + ADDITIONAL_ENTRIES[slug] ||= [] + ADDITIONAL_ENTRIES[slug] << [name, id] + end + end + + ADDITIONAL_ENTRIES[slug] || [] + end + + def include_default_entry? + subpath != 'keyword_index.html' && subpath != 'sitemap.html' + end + end + end +end diff --git a/lib/docs/scrapers/sqlite.rb b/lib/docs/scrapers/sqlite.rb new file mode 100644 index 00000000..f992c8f1 --- /dev/null +++ b/lib/docs/scrapers/sqlite.rb @@ -0,0 +1,52 @@ +module Docs + class Sqlite < FileScraper + self.name = 'SQLite' + self.type = 'sqlite' + self.release = '3.15.2' + self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/' + self.base_url = 'https://sqlite.org/' + self.root_path = 'docs.html' + self.initial_paths = %w(keyword_index.html) + self.links = { + home: 'https://sqlite.org/', + code: 'https://www.sqlite.org/src/' + } + + html_filters.insert_before 'clean_html', 'sqlite/clean_js_tables' + html_filters.push 'sqlite/entries', 'sqlite/clean_html' + + options[:only_patterns] = [/\.html\z/] + options[:skip_patterns] = [/releaselog/, /consortium/] + options[:skip] = %w( + index.html + about.html + download.html + copyright.html + support.html + prosupport.html + hp1.html + news.html + oldnews.html + doclist.html + dev.html + chronology.html + not-found.html + famous.html + books.html + crew.html + mostdeployed.html + requirements.html + session/intro.html + syntax.html + ) + + options[:attribution] = 'SQLite is in the Public Domain.' + + private + + def parse(html) + html.gsub! %r{(]*>[^<]+)}, '\1' + super + end + end +end diff --git a/public/icons/docs/sqlite/16.png b/public/icons/docs/sqlite/16.png new file mode 100644 index 00000000..9a503f44 Binary files /dev/null and b/public/icons/docs/sqlite/16.png differ diff --git a/public/icons/docs/sqlite/16@2x.png b/public/icons/docs/sqlite/16@2x.png new file mode 100644 index 00000000..2b0d1e17 Binary files /dev/null and b/public/icons/docs/sqlite/16@2x.png differ diff --git a/test/lib/docs/core/parser_test.rb b/test/lib/docs/core/parser_test.rb index 240d0156..1a83a02b 100644 --- a/test/lib/docs/core/parser_test.rb +++ b/test/lib/docs/core/parser_test.rb @@ -28,4 +28,16 @@ class DocsParserTest < MiniTest::Spec end end end + + describe "#title" do + it "returns nil when there is no " do + body = '<!doctype html><meta charset=utf-8><div>Test</div>' + assert_nil parser(body).title + end + + it "returns the <title> when there is one" do + body = '<!doctype html><meta charset=utf-8><title>Title
Test
' + assert_equal 'Title', parser(body).title + end + end end