diff --git a/.gitignore b/.gitignore index 8b222826..a2e89741 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ public/fonts public/docs/**/* !public/docs/docs.json !public/docs/**/index.json +log diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee index fef9a024..e9d6a748 100644 --- a/assets/javascripts/templates/pages/about_tmpl.coffee +++ b/assets/javascripts/templates/pages/about_tmpl.coffee @@ -410,6 +410,11 @@ credits = [ '2006-2016 LÖVE Development Team', 'GFDL', 'http://www.gnu.org/copyleft/fdl.html' + ], [ + 'MariaDB', + '2018 MariaDB', + 'CC BY-SA & GFDL', + 'https://mariadb.com/kb/en/library/documentation/+license/' ], [ 'Marionette.js', '2017 Muted Solutions, LLC', diff --git a/assets/stylesheets/application-dark.css.scss b/assets/stylesheets/application-dark.css.scss index 4500f90f..9f6b3a7d 100644 --- a/assets/stylesheets/application-dark.css.scss +++ b/assets/stylesheets/application-dark.css.scss @@ -64,6 +64,7 @@ 'pages/liquid', 'pages/love', 'pages/lua', + 'pages/mariadb', 'pages/mdn', 'pages/meteor', 'pages/modernizr', diff --git a/assets/stylesheets/application.css.scss b/assets/stylesheets/application.css.scss index f7321135..64b85f1f 100644 --- a/assets/stylesheets/application.css.scss +++ b/assets/stylesheets/application.css.scss @@ -64,6 +64,7 @@ 'pages/liquid', 'pages/love', 'pages/lua', + 'pages/mariadb', 'pages/mdn', 'pages/meteor', 'pages/modernizr', diff --git a/assets/stylesheets/pages/_mariadb.scss b/assets/stylesheets/pages/_mariadb.scss new file mode 100644 index 00000000..ef6144ce --- /dev/null +++ b/assets/stylesheets/pages/_mariadb.scss @@ -0,0 +1,7 @@ +._mariadb { + @extend %simple; + + .graybox { + @extend %note; + } +} diff --git a/lib/docs/filters/mariadb/clean_html.rb b/lib/docs/filters/mariadb/clean_html.rb new file mode 100644 index 00000000..ffbbb160 --- /dev/null +++ b/lib/docs/filters/mariadb/clean_html.rb @@ -0,0 +1,71 @@ +require 'net/http' + +module Docs + class Mariadb + class CleanHtmlFilter < Filter + @@known_urls = Hash.new + + def call + # Extract main content + @doc = at_css('#content') + + # Remove navigation at the bottom + css('.simple_section_nav').remove + + # Remove table of contents + css('.table_of_contents').remove + + # Add code highlighting and remove nested tags + css('pre').each do |node| + node.content = node.content + node['data-language'] = 'sql' + end + + # Fix links like http://kb-mirror.mariadb.com/kb/en/bitwise-or/ to not redirect to an external page + css('a').each do |node| + url = node['href'] + + if /^http:\/\/kb-mirror\.mariadb\.com\/kb\/en\/[^\/]+\/(#[^\/]+)?$/.match?(url) + final_url = get_final_url(url) + + if !final_url.nil? && final_url.start_with?('/kb/en/library/documentation/') + node['href'] = "#{'../' * subpath.count('/')}#{final_url[29..-1]}index" + end + end + end + + # Remove navigation items containing only numbers + css('.node_comments').each do |node| + if node.content.scan(/\D/).empty? + node.remove + end + end + + # Convert listings (pages like http://kb-mirror.mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables + css('ul.listing').each do |node| + rows = [] + + node.css('li').each do |li| + name = li.at_css('.media-heading').content + description = li.at_css('.blurb').content + url = li.at_css('a')['href'] + rows << "#{name}#{description}" + end + + table = "#{rows.join('')}
TitleDescription
" + node.replace(table) + end + + doc + end + + def get_final_url(url) + unless @@known_urls.has_key?(url) + @@known_urls[url] = Net::HTTP.get_response(URI(url))['location'] + end + + @@known_urls[url] + end + end + end +end diff --git a/lib/docs/filters/mariadb/entries.rb b/lib/docs/filters/mariadb/entries.rb new file mode 100644 index 00000000..32d4f6b2 --- /dev/null +++ b/lib/docs/filters/mariadb/entries.rb @@ -0,0 +1,14 @@ +module Docs + class Mariadb + class EntriesFilter < Docs::EntriesFilter + def get_name + at_css('.container > h1').content.strip + end + + def get_type + link = at_css('#breadcrumbs > a:nth-child(6)') + link.nil? ? at_css('#breadcrumbs > a:nth-child(5)').content : link.content + end + end + end +end diff --git a/lib/docs/scrapers/mariadb.rb b/lib/docs/scrapers/mariadb.rb new file mode 100644 index 00000000..b4293a9a --- /dev/null +++ b/lib/docs/scrapers/mariadb.rb @@ -0,0 +1,22 @@ +module Docs + class Mariadb < UrlScraper + self.name = 'MariaDB' + self.type = 'mariadb' + self.release = '10.3.8' + self.base_url = 'http://kb-mirror.mariadb.com/kb/en/library/documentation/' + self.links = { + home: 'https://mariadb.com/', + code: 'https://github.com/MariaDB/server' + } + + html_filters.push 'mariadb/entries', 'mariadb/clean_html', 'title' + + options[:download_images] = false + options[:root_title] = 'MariaDB' + + options[:attribution] = <<-HTML + © 2018 MariaDB
+ Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License. + HTML + end +end