From 50e138bd67b4002083d25942d9d5ebfc1475d05d Mon Sep 17 00:00:00 2001 From: Thibaut Courouble Date: Sun, 2 Oct 2016 11:13:41 -0400 Subject: [PATCH] Update PostgreSQL documentation (9.6.0, 9.5.4, 9.4.9) --- assets/javascripts/collections/types.coffee | 3 ++ assets/stylesheets/pages/_postgres.scss | 9 +++- lib/docs/filters/postgresql/clean_html.rb | 15 +++++-- lib/docs/filters/postgresql/entries.rb | 41 +++++++++++++++---- .../filters/postgresql/extract_metadata.rb | 10 +++-- lib/docs/scrapers/postgresql.rb | 30 ++++++++++---- 6 files changed, 84 insertions(+), 24 deletions(-) diff --git a/assets/javascripts/collections/types.coffee b/assets/javascripts/collections/types.coffee index 0f5cbd9a..83a68979 100644 --- a/assets/javascripts/collections/types.coffee +++ b/assets/javascripts/collections/types.coffee @@ -8,9 +8,12 @@ class app.collections.Types extends app.Collection result.filter (e) -> e.length > 0 GUIDES_RGX = /(^|\()(guides?|tutorials?|reference|book|getting\ started|manual)($|[\):])/i + APPENDIX_RGX = /appendix/i _groupFor: (type) -> if GUIDES_RGX.test(type.name) 0 + else if APPENDIX_RGX.test(type.name) + 2 else 1 diff --git a/assets/stylesheets/pages/_postgres.scss b/assets/stylesheets/pages/_postgres.scss index 5694e4fb..4f853bf3 100644 --- a/assets/stylesheets/pages/_postgres.scss +++ b/assets/stylesheets/pages/_postgres.scss @@ -6,9 +6,14 @@ .VARIABLELIST dt { @extend %block-label, %label-blue; } - blockquote.NOTE, blockquote.IMPORTANT { @extend %note; } - blockquote.TIP { @extend %note, %note-green; } + blockquote.NOTE, blockquote.IMPORTANT, blockquote.TIP, blockquote.CAUTION { @extend %note; } + blockquote.TIP { @extend %note-green; } + blockquote.CAUTION { @extend %note-orange; } p > code { @extend %label; } p.c2 { font-weight: bold; } + + .NAVFOOTER > table { width: 100%; } + td[align=center] { text-align: center; } + td[align=right] { text-align: right; } } diff --git a/lib/docs/filters/postgresql/clean_html.rb b/lib/docs/filters/postgresql/clean_html.rb index cf2351c8..62ba40ec 100644 --- a/lib/docs/filters/postgresql/clean_html.rb +++ b/lib/docs/filters/postgresql/clean_html.rb @@ -13,24 +13,33 @@ module Docs def other @doc = at_css('#docContent') - css('.NAVHEADER', '.NAVFOOTER').remove + css('.NAVHEADER', 'hr', '.NAVFOOTER a[accesskey="H"]').remove css('a[name]').each do |node| node.parent['id'] = node['name'] node.before(node.children).remove end - css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER').each do |node| + css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER', 'div.NOTE', '.APPENDIX').each do |node| node.before(node.children).remove end + css('div.CAUTION table.CAUTION').each do |node| + parent = node.parent + title = node.at_css('.c2, .c3, .c4, .c5').content + node.replace(node.css('p')) + parent.first_element_child.inner_html = "#{title}: #{parent.first_element_child.inner_html}" + parent.name = 'blockquote' + end + css('table').each do |node| node.remove_attribute 'border' node.remove_attribute 'width' + node.remove_attribute 'cellspacing' + node.remove_attribute 'cellpadding' end css('td').each do |node| - node.remove_attribute 'align' node.remove_attribute 'valign' end diff --git a/lib/docs/filters/postgresql/entries.rb b/lib/docs/filters/postgresql/entries.rb index 83d5aeee..b49326a8 100644 --- a/lib/docs/filters/postgresql/entries.rb +++ b/lib/docs/filters/postgresql/entries.rb @@ -4,6 +4,7 @@ module Docs REPLACE_NAMES = { 'Sorting Rows' => 'ORDER BY', 'Select Lists' => 'SELECT Lists', + 'Comparison Functions and Operators' => 'Comparisons', 'Data Type Formatting Functions' => 'Formatting Functions', 'Enum Support Functions' => 'Enum Functions', 'Row and Array Comparisons' => 'Array Comparisons', @@ -43,8 +44,14 @@ module Docs @base_name ||= clean_heading_name(at_css('h1').content) end + def heading_level + @heading_level ||= at_css('h1').content.scan(/\d+(?=\.)/).last + end + def get_name - if %w(Overview Introduction).include?(base_name) + if type.start_with?('Tutorial') + "#{heading_level}. #{base_name}" + elsif %w(Overview Introduction).include?(base_name) result[:pg_chapter_name] elsif PREPEND_TYPES.include?(type) || type.start_with?('Internals') "#{type.remove('Internals: ')}: #{base_name}" @@ -58,17 +65,23 @@ module Docs if result[:pg_up_path] == 'sql-commands.html' 'Commands' + elsif result[:pg_up_path] == 'appendixes.html' + 'Appendixes' elsif result[:pg_up_path].start_with?('reference-') 'Applications' elsif type = result[:pg_chapter_name] - if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/)) + if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum|Comp)(.+) Func/)) "Functions: #{match[1]}" else type.remove! %r{\ASQL } type = REPLACE_TYPES[type] || type - type = "Internals: #{type}" if INTERNAL_TYPES.include?(type) + type.prepend 'Internals: ' if INTERNAL_TYPES.include?(type) + type.prepend 'Tutorial: ' if slug.start_with?('tutorial') type end + elsif type = result[:pg_appendix_name] + type.prepend 'Appendix: ' + type end end @@ -97,9 +110,15 @@ module Docs entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code') else if type && type.start_with?('Functions') - entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child') - entries.concat get_custom_entries('> .TABLE td:first-child > p > code:first-child') - entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } if slug == 'functions-comparison' + entries.concat get_custom_entries('> .TABLE td:first-child > code.LITERAL:first-child') + entries.concat get_custom_entries('> .TABLE td:first-child > code.FUNCTION:first-child') + entries.concat get_custom_entries('> .TABLE td:first-child > code:not(.LITERAL):first-child + code.LITERAL') + entries.concat get_custom_entries('> .TABLE td:first-child > p > code.LITERAL:first-child') + entries.concat get_custom_entries('> .TABLE td:first-child > p > code.FUNCTION:first-child') + entries.concat get_custom_entries('> .TABLE td:first-child > p > code:not(.LITERAL):first-child + code.LITERAL') + if slug == 'functions-comparison' && !at_css('#FUNCTIONS-COMPARISON-PRED-TABLE') # before 9.6 + entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } + end end end @@ -157,12 +176,18 @@ module Docs def skip_additional_entries? return true unless type - SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type) || type.start_with?('Internals') + SKIP_ENTRIES_SLUGS.include?(slug) || + SKIP_ENTRIES_TYPES.include?(type) || + type.start_with?('Internals') || + type.start_with?('Tutorial') || + type.start_with?('Appendix') end def clean_heading_name(name) name.remove! 'Chapter ' name.remove! %r{\A[\d\.\s]+} + name.remove! 'Appendix ' + name.remove! %r{\A[A-Z]\.[\d\.\s]*} name.remove! 'Using ' name.remove! %r{\AThe } name.remove! ' (Common Table Expressions)' @@ -185,7 +210,7 @@ module Docs name.squeeze! ' ' name.remove! %r{\([^\)]*\z} # bug fix: json_populate_record name = '||' if name.include? ' || ' - id = name.gsub(/[^a-z0-9\-_]/) { |char| char.ord } + id = name.gsub(/[^a-zA-Z0-9\-_]/) { |char| char.ord } id = id.parameterize name.prepend "#{additional_entry_prefix}: " diff --git a/lib/docs/filters/postgresql/extract_metadata.rb b/lib/docs/filters/postgresql/extract_metadata.rb index 186e2c13..ba313dca 100644 --- a/lib/docs/filters/postgresql/extract_metadata.rb +++ b/lib/docs/filters/postgresql/extract_metadata.rb @@ -15,9 +15,13 @@ module Docs def extract_chapter return unless text = at_css('.NAVHEADER td[align="center"]').content - return unless match = text.match(/\AChapter (\d+)\. (.+)\z/) - result[:pg_chapter] = match[1].to_i - result[:pg_chapter_name] = match[2].strip + if match = text.match(/\AChapter (\d+)\. (.+)\z/) + result[:pg_chapter] = match[1].to_i + result[:pg_chapter_name] = match[2].strip + elsif match = text.match(/\AAppendix ([A-Z])\. (.+)\z/) + result[:pg_appendix] = match[1] + result[:pg_appendix_name] = match[2].strip + end end end end diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb index a2d0c480..e8cbde83 100644 --- a/lib/docs/scrapers/postgresql.rb +++ b/lib/docs/scrapers/postgresql.rb @@ -5,7 +5,7 @@ module Docs self.name = 'PostgreSQL' self.type = 'postgres' self.root_path = 'reference.html' - self.initial_paths = %w(sql.html admin.html internals.html) + self.initial_paths = %w(sql.html admin.html internals.html appendixes.html tutorial.html) html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata' html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title' @@ -23,10 +23,13 @@ module Docs supported-platforms.html error-message-reporting.html error-style-guide.html - plhandler.html) + plhandler.html + sourcerepo.html + git.html + bug-reporting.html + client-interfaces.html) options[:skip_patterns] = [ - /\Atutorial/, /\Ainstall/, /\Aregress/, /\Aprotocol/, @@ -35,21 +38,32 @@ module Docs /\Afdw/, /\Atablesample/, /\Acustom-scan/, - /\Abki/ ] + /\Abki/, + /\Arelease/, + /\Acontrib-prog/, + /\Aexternal/, + /\Adocguide/, + /\Afeatures/, + /\Aunsupported-features/ ] options[:attribution] = <<-HTML © 1996–2016 The PostgreSQL Global Development Group
Licensed under the PostgreSQL License. HTML + version '9.6' do + self.release = '9.6.0' + self.base_url = 'https://www.postgresql.org/docs/9.6/static/' + end + version '9.5' do - self.release = '9.5' - self.base_url = 'http://www.postgresql.org/docs/9.5/static/' + self.release = '9.5.4' + self.base_url = 'https://www.postgresql.org/docs/9.5/static/' end version '9.4' do - self.release = '9.4' - self.base_url = 'http://www.postgresql.org/docs/9.4/static/' + self.release = '9.4.9' + self.base_url = 'https://www.postgresql.org/docs/9.4/static/' end end end