From 426d6b8a4d26c3401dddd97e7e92003c1c3890b4 Mon Sep 17 00:00:00 2001 From: Thibaut Courouble Date: Sat, 13 Feb 2016 12:00:39 -0500 Subject: [PATCH] Improve & version PostgreSQL documentation (9.4 and 9.5) --- .../templates/pages/about_tmpl.coffee | 2 +- assets/stylesheets/pages/_postgres.scss | 1 + lib/docs/filters/postgresql/clean_html.rb | 2 +- lib/docs/filters/postgresql/entries.rb | 44 ++++++++-- .../filters/postgresql/extract_metadata.rb | 2 +- lib/docs/scrapers/postgresql.rb | 83 +++++++------------ 6 files changed, 73 insertions(+), 61 deletions(-) diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee index d8cd433f..58a56225 100644 --- a/assets/javascripts/templates/pages/about_tmpl.coffee +++ b/assets/javascripts/templates/pages/about_tmpl.coffee @@ -321,7 +321,7 @@ credits = [ 'https://creativecommons.org/licenses/by/3.0/' ], [ 'PostgreSQL', - '1996-2013 The PostgreSQL Global Development Group
© 1994 The Regents of the University of California', + '1996-2016 The PostgreSQL Global Development Group
© 1994 The Regents of the University of California', 'PostgreSQL', 'http://www.postgresql.org/about/licence/' ], [ diff --git a/assets/stylesheets/pages/_postgres.scss b/assets/stylesheets/pages/_postgres.scss index da8dc08a..5694e4fb 100644 --- a/assets/stylesheets/pages/_postgres.scss +++ b/assets/stylesheets/pages/_postgres.scss @@ -10,4 +10,5 @@ blockquote.TIP { @extend %note, %note-green; } p > code { @extend %label; } + p.c2 { font-weight: bold; } } diff --git a/lib/docs/filters/postgresql/clean_html.rb b/lib/docs/filters/postgresql/clean_html.rb index a30e4543..afdb8b68 100644 --- a/lib/docs/filters/postgresql/clean_html.rb +++ b/lib/docs/filters/postgresql/clean_html.rb @@ -20,7 +20,7 @@ module Docs node.before(node.children).remove end - css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt').each do |node| + css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER').each do |node| node.before(node.children).remove end diff --git a/lib/docs/filters/postgresql/entries.rb b/lib/docs/filters/postgresql/entries.rb index c39ee4a2..83d5aeee 100644 --- a/lib/docs/filters/postgresql/entries.rb +++ b/lib/docs/filters/postgresql/entries.rb @@ -23,7 +23,21 @@ module Docs 'High Availability, Load Balancing, and Replication' => 'High Availability', 'Monitoring Database Activity' => 'Monitoring', 'Monitoring Disk Usage' => 'Monitoring', - 'Reliability and the Write-Ahead Log' => 'Write-Ahead Log' } + 'Reliability and the Write-Ahead Log' => 'Write-Ahead Log', + 'Overview of PostgreSQL Internals' => 'Internals', + 'System Catalogs' => 'Internals: Catalogs', + 'How the Planner Uses Statistics' => 'Internals', + 'Index Access Method Interface Definition' => 'Index Access Method', + 'Database Physical Storage' => 'Physical Storage' } + + INTERNAL_TYPES = [ + 'Genetic Query Optimizer', + 'Index Access Method', + 'GiST Indexes', + 'SP-GiST Indexes', + 'GIN Indexes', + 'BRIN Indexes', + 'Physical Storage' ] def base_name @base_name ||= clean_heading_name(at_css('h1').content) @@ -32,8 +46,8 @@ module Docs def get_name if %w(Overview Introduction).include?(base_name) result[:pg_chapter_name] - elsif PREPEND_TYPES.include?(type) - "#{type}: #{base_name}" + elsif PREPEND_TYPES.include?(type) || type.start_with?('Internals') + "#{type.remove('Internals: ')}: #{base_name}" else REPLACE_NAMES[base_name] || base_name end @@ -50,8 +64,10 @@ module Docs if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/)) "Functions: #{match[1]}" else - type.remove! 'SQL ' - REPLACE_TYPES[type] || type + type.remove! %r{\ASQL } + type = REPLACE_TYPES[type] || type + type = "Internals: #{type}" if INTERNAL_TYPES.include?(type) + type end end end @@ -60,6 +76,7 @@ module Docs return [] if skip_additional_entries? return config_additional_entries if type && type.include?('Configuration') return data_types_additional_entries if type == 'Data Types' + return command_additional_entries if type == 'Commands' return get_heading_entries('h3[id]') if slug == 'functions-xml' entries = get_heading_entries('h2[id]') @@ -81,6 +98,7 @@ module Docs else if type && type.start_with?('Functions') entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child') + entries.concat get_custom_entries('> .TABLE td:first-child > p > code:first-child') entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } if slug == 'functions-comparison' end end @@ -107,8 +125,18 @@ module Docs get_custom_entries(selector) end + def command_additional_entries + css('.REFSECT2[id^="SQL"]').each_with_object([]) do |node, entries| + next unless heading = node.at_css('h3') + next unless heading.content.strip =~ /[A-Z_\-]+ Clause/ + name = heading.at_css('.LITERAL').content + name.prepend "#{self.name} ... " + entries << [name, node['id']] + end + end + def include_default_entry? - !initial_page? && !at_css('.TOC') + !initial_page? && !at_css('.TOC') && type end SKIP_ENTRIES_SLUGS = [ @@ -128,10 +156,12 @@ module Docs 'Monitoring' ] def skip_additional_entries? - SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type) + return true unless type + SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type) || type.start_with?('Internals') end def clean_heading_name(name) + name.remove! 'Chapter ' name.remove! %r{\A[\d\.\s]+} name.remove! 'Using ' name.remove! %r{\AThe } diff --git a/lib/docs/filters/postgresql/extract_metadata.rb b/lib/docs/filters/postgresql/extract_metadata.rb index 50e15d87..186e2c13 100644 --- a/lib/docs/filters/postgresql/extract_metadata.rb +++ b/lib/docs/filters/postgresql/extract_metadata.rb @@ -17,7 +17,7 @@ module Docs return unless text = at_css('.NAVHEADER td[align="center"]').content return unless match = text.match(/\AChapter (\d+)\. (.+)\z/) result[:pg_chapter] = match[1].to_i - result[:pg_chapter_name] = match[2] + result[:pg_chapter_name] = match[2].strip end end end diff --git a/lib/docs/scrapers/postgresql.rb b/lib/docs/scrapers/postgresql.rb index a6d967e3..a2d0c480 100644 --- a/lib/docs/scrapers/postgresql.rb +++ b/lib/docs/scrapers/postgresql.rb @@ -1,11 +1,11 @@ module Docs class Postgresql < UrlScraper + include FixInternalUrlsBehavior + self.name = 'PostgreSQL' self.type = 'postgres' - self.release = '9.4' - self.base_url = "http://www.postgresql.org/docs/#{release}/static/" self.root_path = 'reference.html' - self.initial_paths = %w(sql.html admin.html) + self.initial_paths = %w(sql.html admin.html internals.html) html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata' html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title' @@ -14,61 +14,42 @@ module Docs options[:root_title] = 'PostgreSQL' options[:follow_links] = ->(filter) { filter.initial_page? } - options[:only] = %w( - arrays.html - rowtypes.html - rangetypes.html - transaction-iso.html - explicit-locking.html - applevel-consistency.html - locking-indexes.html - config-setting.html - locale.html - collation.html - multibyte.html - using-explain.html - planner-stats.html - explicit-joins.html - populate.html - non-durability.html - logfile-maintenance.html - continuous-archiving.html - dynamic-trace.html) - - options[:only_patterns] = [ - /\Asql\-/, - /\Aapp\-/, - /\Addl\-/, - /\Adml\-/, - /\Aqueries\-/, - /\Adatatype\-/, - /\Afunctions\-/, - /\Atypeconv\-/, - /\Atextsearch\-/, - /\Amvcc\-/, - /\Aindexes\-/, - /\Aruntime\-config\-/, - /\Aauth\-/, - /\Aclient\-authentication/, - /\Amanage\-ag/, - /\Aroutine/, - /\Abackup\-/, - /\Amonitoring\-/, - /\Awal\-/, - /\Adisk/, - /role/, - /recovery/, - /standby/] - options[:skip] = %w( + index.html ddl-others.html functions-event-triggers.html functions-trigger.html - textsearch-migration.html) + textsearch-migration.html + supported-platforms.html + error-message-reporting.html + error-style-guide.html + plhandler.html) + + options[:skip_patterns] = [ + /\Atutorial/, + /\Ainstall/, + /\Aregress/, + /\Aprotocol/, + /\Asource/, + /\Anls/, + /\Afdw/, + /\Atablesample/, + /\Acustom-scan/, + /\Abki/ ] options[:attribution] = <<-HTML - © 1996–2014 The PostgreSQL Global Development Group
+ © 1996–2016 The PostgreSQL Global Development Group
Licensed under the PostgreSQL License. HTML + + version '9.5' do + self.release = '9.5' + self.base_url = 'http://www.postgresql.org/docs/9.5/static/' + end + + version '9.4' do + self.release = '9.4' + self.base_url = 'http://www.postgresql.org/docs/9.4/static/' + end end end