Improve & version PostgreSQL documentation (9.4 and 9.5)

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -321,7 +321,7 @@ credits = [
     'https://creativecommons.org/licenses/by/3.0/'
   ], [
     'PostgreSQL',
-    '1996-2013 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
+    '1996-2016 The PostgreSQL Global Development Group<br>&copy; 1994 The Regents of the University of California',
     'PostgreSQL',
     'http://www.postgresql.org/about/licence/'
   ], [

assets/stylesheets/pages/_postgres.scss

@@ -10,4 +10,5 @@
   blockquote.TIP { @extend %note, %note-green; }
 
   p > code { @extend %label; }
+  p.c2 { font-weight: bold; }
 }

lib/docs/filters/postgresql/clean_html.rb

@@ -20,7 +20,7 @@ module Docs
           node.before(node.children).remove
         end
 
-        css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt').each do |node|
+        css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER').each do |node|
           node.before(node.children).remove
         end
 

lib/docs/filters/postgresql/entries.rb

@@ -23,7 +23,21 @@ module Docs
         'High Availability, Load Balancing, and Replication' => 'High Availability',
         'Monitoring Database Activity' => 'Monitoring',
         'Monitoring Disk Usage' => 'Monitoring',
-        'Reliability and the Write-Ahead Log' => 'Write-Ahead Log' }
+        'Reliability and the Write-Ahead Log' => 'Write-Ahead Log',
+        'Overview of PostgreSQL Internals' => 'Internals',
+        'System Catalogs' => 'Internals: Catalogs',
+        'How the Planner Uses Statistics' => 'Internals',
+        'Index Access Method Interface Definition' => 'Index Access Method',
+        'Database Physical Storage' => 'Physical Storage' }
+
+      INTERNAL_TYPES = [
+        'Genetic Query Optimizer',
+        'Index Access Method',
+        'GiST Indexes',
+        'SP-GiST Indexes',
+        'GIN Indexes',
+        'BRIN Indexes',
+        'Physical Storage' ]
 
       def base_name
         @base_name ||= clean_heading_name(at_css('h1').content)
@@ -32,8 +46,8 @@ module Docs
       def get_name
         if %w(Overview Introduction).include?(base_name)
           result[:pg_chapter_name]
-        elsif PREPEND_TYPES.include?(type)
+        elsif PREPEND_TYPES.include?(type) || type.start_with?('Internals')
-          "#{type}: #{base_name}"
+          "#{type.remove('Internals: ')}: #{base_name}"
         else
           REPLACE_NAMES[base_name] || base_name
         end
@@ -50,8 +64,10 @@ module Docs
           if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/))
             "Functions: #{match[1]}"
           else
-            type.remove! 'SQL '
+            type.remove! %r{\ASQL }
-            REPLACE_TYPES[type] || type
+            type = REPLACE_TYPES[type] || type
+            type = "Internals: #{type}" if INTERNAL_TYPES.include?(type)
+            type
           end
         end
       end
@@ -60,6 +76,7 @@ module Docs
         return [] if skip_additional_entries?
         return config_additional_entries if type && type.include?('Configuration')
         return data_types_additional_entries if type == 'Data Types'
+        return command_additional_entries if type == 'Commands'
         return get_heading_entries('h3[id]') if slug == 'functions-xml'
 
         entries = get_heading_entries('h2[id]')
@@ -81,6 +98,7 @@ module Docs
         else
           if type && type.start_with?('Functions')
             entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')
+            entries.concat get_custom_entries('> .TABLE td:first-child > p > code:first-child')
             entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } if slug == 'functions-comparison'
           end
         end
@@ -107,8 +125,18 @@ module Docs
         get_custom_entries(selector)
       end
 
+      def command_additional_entries
+        css('.REFSECT2[id^="SQL"]').each_with_object([]) do |node, entries|
+          next unless heading = node.at_css('h3')
+          next unless heading.content.strip =~ /[A-Z_\-]+ Clause/
+          name = heading.at_css('.LITERAL').content
+          name.prepend "#{self.name} ... "
+          entries << [name, node['id']]
+        end
+      end
+
       def include_default_entry?
-        !initial_page? && !at_css('.TOC')
+        !initial_page? && !at_css('.TOC') && type
       end
 
       SKIP_ENTRIES_SLUGS = [
@@ -128,10 +156,12 @@ module Docs
         'Monitoring' ]
 
       def skip_additional_entries?
-        SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type)
+        return true unless type
+        SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type) || type.start_with?('Internals')
       end
 
       def clean_heading_name(name)
+        name.remove! 'Chapter '
         name.remove! %r{\A[\d\.\s]+}
         name.remove! 'Using '
         name.remove! %r{\AThe }

lib/docs/filters/postgresql/extract_metadata.rb

@@ -17,7 +17,7 @@ module Docs
         return unless text = at_css('.NAVHEADER td[align="center"]').content
         return unless match = text.match(/\AChapter (\d+)\. (.+)\z/)
         result[:pg_chapter] = match[1].to_i
-        result[:pg_chapter_name] = match[2]
+        result[:pg_chapter_name] = match[2].strip
       end
     end
   end

lib/docs/scrapers/postgresql.rb

@@ -1,11 +1,11 @@
 module Docs
   class Postgresql < UrlScraper
+    include FixInternalUrlsBehavior
+
     self.name = 'PostgreSQL'
     self.type = 'postgres'
-    self.release = '9.4'
-    self.base_url = "http://www.postgresql.org/docs/#{release}/static/"
     self.root_path = 'reference.html'
-    self.initial_paths = %w(sql.html admin.html)
+    self.initial_paths = %w(sql.html admin.html internals.html)
 
     html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata'
     html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title'
@@ -14,61 +14,42 @@ module Docs
     options[:root_title] = 'PostgreSQL'
     options[:follow_links] = ->(filter) { filter.initial_page? }
 
-    options[:only] = %w(
-      arrays.html
-      rowtypes.html
-      rangetypes.html
-      transaction-iso.html
-      explicit-locking.html
-      applevel-consistency.html
-      locking-indexes.html
-      config-setting.html
-      locale.html
-      collation.html
-      multibyte.html
-      using-explain.html
-      planner-stats.html
-      explicit-joins.html
-      populate.html
-      non-durability.html
-      logfile-maintenance.html
-      continuous-archiving.html
-      dynamic-trace.html)
-
-    options[:only_patterns] = [
-      /\Asql\-/,
-      /\Aapp\-/,
-      /\Addl\-/,
-      /\Adml\-/,
-      /\Aqueries\-/,
-      /\Adatatype\-/,
-      /\Afunctions\-/,
-      /\Atypeconv\-/,
-      /\Atextsearch\-/,
-      /\Amvcc\-/,
-      /\Aindexes\-/,
-      /\Aruntime\-config\-/,
-      /\Aauth\-/,
-      /\Aclient\-authentication/,
-      /\Amanage\-ag/,
-      /\Aroutine/,
-      /\Abackup\-/,
-      /\Amonitoring\-/,
-      /\Awal\-/,
-      /\Adisk/,
-      /role/,
-      /recovery/,
-      /standby/]
-
     options[:skip] = %w(
+      index.html
       ddl-others.html
       functions-event-triggers.html
       functions-trigger.html
-      textsearch-migration.html)
+      textsearch-migration.html
+      supported-platforms.html
+      error-message-reporting.html
+      error-style-guide.html
+      plhandler.html)
+
+    options[:skip_patterns] = [
+      /\Atutorial/,
+      /\Ainstall/,
+      /\Aregress/,
+      /\Aprotocol/,
+      /\Asource/,
+      /\Anls/,
+      /\Afdw/,
+      /\Atablesample/,
+      /\Acustom-scan/,
+      /\Abki/ ]
 
     options[:attribution] = <<-HTML
-      &copy; 1996&ndash;2014 The PostgreSQL Global Development Group<br>
+      &copy; 1996&ndash;2016 The PostgreSQL Global Development Group<br>
       Licensed under the PostgreSQL License.
     HTML
+
+    version '9.5' do
+      self.release = '9.5'
+      self.base_url = 'http://www.postgresql.org/docs/9.5/static/'
+    end
+
+    version '9.4' do
+      self.release = '9.4'
+      self.base_url = 'http://www.postgresql.org/docs/9.4/static/'
+    end
   end
 end