Update PostgreSQL documentation (9.6.0, 9.5.4, 9.4.9)

assets/javascripts/collections/types.coffee

@@ -8,9 +8,12 @@ class app.collections.Types extends app.Collection
     result.filter (e) -> e.length > 0
 
   GUIDES_RGX = /(^|\()(guides?|tutorials?|reference|book|getting\ started|manual)($|[\):])/i
+  APPENDIX_RGX = /appendix/i
 
   _groupFor: (type) ->
     if GUIDES_RGX.test(type.name)
       0
+    else if APPENDIX_RGX.test(type.name)
+      2
     else
       1

assets/stylesheets/pages/_postgres.scss

@@ -6,9 +6,14 @@
 
   .VARIABLELIST dt { @extend %block-label, %label-blue; }
 
-  blockquote.NOTE, blockquote.IMPORTANT { @extend %note; }
+  blockquote.NOTE, blockquote.IMPORTANT, blockquote.TIP, blockquote.CAUTION { @extend %note; }
-  blockquote.TIP { @extend %note, %note-green; }
+  blockquote.TIP { @extend %note-green; }
+  blockquote.CAUTION { @extend %note-orange; }
 
   p > code { @extend %label; }
   p.c2 { font-weight: bold; }
+
+  .NAVFOOTER > table { width: 100%; }
+  td[align=center] { text-align: center; }
+  td[align=right] { text-align: right; }
 }

lib/docs/filters/postgresql/clean_html.rb

@@ -13,24 +13,33 @@ module Docs
       def other
         @doc = at_css('#docContent')
 
-        css('.NAVHEADER', '.NAVFOOTER').remove
+        css('.NAVHEADER', 'hr', '.NAVFOOTER a[accesskey="H"]').remove
 
         css('a[name]').each do |node|
           node.parent['id'] = node['name']
           node.before(node.children).remove
         end
 
-        css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER').each do |node|
+        css('div.SECT1', 'pre > kbd', 'tt > code', 'h1 > tt', '> .CHAPTER', 'div.NOTE', '.APPENDIX').each do |node|
           node.before(node.children).remove
         end
 
+        css('div.CAUTION table.CAUTION').each do |node|
+          parent = node.parent
+          title = node.at_css('.c2, .c3, .c4, .c5').content
+          node.replace(node.css('p'))
+          parent.first_element_child.inner_html = "<strong>#{title}:</strong> #{parent.first_element_child.inner_html}"
+          parent.name = 'blockquote'
+        end
+
         css('table').each do |node|
           node.remove_attribute 'border'
           node.remove_attribute 'width'
+          node.remove_attribute 'cellspacing'
+          node.remove_attribute 'cellpadding'
         end
 
         css('td').each do |node|
-          node.remove_attribute 'align'
           node.remove_attribute 'valign'
         end
 

lib/docs/filters/postgresql/entries.rb

@@ -4,6 +4,7 @@ module Docs
       REPLACE_NAMES = {
         'Sorting Rows'                    => 'ORDER BY',
         'Select Lists'                    => 'SELECT Lists',
+        'Comparison Functions and Operators' => 'Comparisons',
         'Data Type Formatting Functions'  => 'Formatting Functions',
         'Enum Support Functions'          => 'Enum Functions',
         'Row and Array Comparisons'       => 'Array Comparisons',
@@ -43,8 +44,14 @@ module Docs
         @base_name ||= clean_heading_name(at_css('h1').content)
       end
 
+      def heading_level
+        @heading_level ||= at_css('h1').content.scan(/\d+(?=\.)/).last
+      end
+
       def get_name
-        if %w(Overview Introduction).include?(base_name)
+        if type.start_with?('Tutorial')
+          "#{heading_level}. #{base_name}"
+        elsif %w(Overview Introduction).include?(base_name)
           result[:pg_chapter_name]
         elsif PREPEND_TYPES.include?(type) || type.start_with?('Internals')
           "#{type.remove('Internals: ')}: #{base_name}"
@@ -58,17 +65,23 @@ module Docs
 
         if result[:pg_up_path] == 'sql-commands.html'
           'Commands'
+        elsif result[:pg_up_path] == 'appendixes.html'
+          'Appendixes'
         elsif result[:pg_up_path].start_with?('reference-')
           'Applications'
         elsif type = result[:pg_chapter_name]
-          if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum)(.+) Func/))
+          if type.start_with?('Func') && (match = base_name.match(/\A(?!Form|Seq|Set|Enum|Comp)(.+) Func/))
             "Functions: #{match[1]}"
           else
             type.remove! %r{\ASQL }
             type = REPLACE_TYPES[type] || type
-            type = "Internals: #{type}" if INTERNAL_TYPES.include?(type)
+            type.prepend 'Internals: ' if INTERNAL_TYPES.include?(type)
+            type.prepend 'Tutorial: ' if slug.start_with?('tutorial')
             type
           end
+        elsif type = result[:pg_appendix_name]
+          type.prepend 'Appendix: '
+          type
         end
       end
 
@@ -97,9 +110,15 @@ module Docs
           entries.concat get_custom_entries('> div[id^="FUNC"] td:first-child > code')
         else
           if type && type.start_with?('Functions')
-            entries.concat get_custom_entries('> .TABLE td:first-child > code:first-child')
+            entries.concat get_custom_entries('> .TABLE td:first-child > code.LITERAL:first-child')
-            entries.concat get_custom_entries('> .TABLE td:first-child > p > code:first-child')
+            entries.concat get_custom_entries('> .TABLE td:first-child > code.FUNCTION:first-child')
-            entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] } if slug == 'functions-comparison'
+            entries.concat get_custom_entries('> .TABLE td:first-child > code:not(.LITERAL):first-child + code.LITERAL')
+            entries.concat get_custom_entries('> .TABLE td:first-child > p > code.LITERAL:first-child')
+            entries.concat get_custom_entries('> .TABLE td:first-child > p > code.FUNCTION:first-child')
+            entries.concat get_custom_entries('> .TABLE td:first-child > p > code:not(.LITERAL):first-child + code.LITERAL')
+            if slug == 'functions-comparison' && !at_css('#FUNCTIONS-COMPARISON-PRED-TABLE') # before 9.6
+              entries.concat %w(IS NULL BETWEEN DISTINCT\ FROM).map { |name| ["#{self.name}: #{name}"] }
+            end
           end
         end
 
@@ -157,12 +176,18 @@ module Docs
 
       def skip_additional_entries?
         return true unless type
-        SKIP_ENTRIES_SLUGS.include?(slug) || SKIP_ENTRIES_TYPES.include?(type) || type.start_with?('Internals')
+        SKIP_ENTRIES_SLUGS.include?(slug) ||
+        SKIP_ENTRIES_TYPES.include?(type) ||
+        type.start_with?('Internals') ||
+        type.start_with?('Tutorial') ||
+        type.start_with?('Appendix')
       end
 
       def clean_heading_name(name)
         name.remove! 'Chapter '
         name.remove! %r{\A[\d\.\s]+}
+        name.remove! 'Appendix '
+        name.remove! %r{\A[A-Z]\.[\d\.\s]*}
         name.remove! 'Using '
         name.remove! %r{\AThe }
         name.remove! ' (Common Table Expressions)'
@@ -185,7 +210,7 @@ module Docs
           name.squeeze! ' '
           name.remove! %r{\([^\)]*\z} # bug fix: json_populate_record
           name = '||' if name.include? ' || '
-          id = name.gsub(/[^a-z0-9\-_]/) { |char| char.ord }
+          id = name.gsub(/[^a-zA-Z0-9\-_]/) { |char| char.ord }
           id = id.parameterize
           name.prepend "#{additional_entry_prefix}: "
 

lib/docs/filters/postgresql/extract_metadata.rb

@@ -15,9 +15,13 @@ module Docs
 
       def extract_chapter
         return unless text = at_css('.NAVHEADER td[align="center"]').content
-        return unless match = text.match(/\AChapter (\d+)\. (.+)\z/)
+        if match = text.match(/\AChapter (\d+)\. (.+)\z/)
-        result[:pg_chapter] = match[1].to_i
+          result[:pg_chapter] = match[1].to_i
-        result[:pg_chapter_name] = match[2].strip
+          result[:pg_chapter_name] = match[2].strip
+        elsif match = text.match(/\AAppendix ([A-Z])\. (.+)\z/)
+          result[:pg_appendix] = match[1]
+          result[:pg_appendix_name] = match[2].strip
+        end
       end
     end
   end

lib/docs/scrapers/postgresql.rb

@@ -5,7 +5,7 @@ module Docs
     self.name = 'PostgreSQL'
     self.type = 'postgres'
     self.root_path = 'reference.html'
-    self.initial_paths = %w(sql.html admin.html internals.html)
+    self.initial_paths = %w(sql.html admin.html internals.html appendixes.html tutorial.html)
 
     html_filters.insert_before 'normalize_urls', 'postgresql/extract_metadata'
     html_filters.push 'postgresql/clean_html', 'postgresql/entries', 'title'
@@ -23,10 +23,13 @@ module Docs
       supported-platforms.html
       error-message-reporting.html
       error-style-guide.html
-      plhandler.html)
+      plhandler.html
+      sourcerepo.html
+      git.html
+      bug-reporting.html
+      client-interfaces.html)
 
     options[:skip_patterns] = [
-      /\Atutorial/,
       /\Ainstall/,
       /\Aregress/,
       /\Aprotocol/,
@@ -35,21 +38,32 @@ module Docs
       /\Afdw/,
       /\Atablesample/,
       /\Acustom-scan/,
-      /\Abki/ ]
+      /\Abki/,
+      /\Arelease/,
+      /\Acontrib-prog/,
+      /\Aexternal/,
+      /\Adocguide/,
+      /\Afeatures/,
+      /\Aunsupported-features/ ]
 
     options[:attribution] = <<-HTML
       &copy; 1996&ndash;2016 The PostgreSQL Global Development Group<br>
       Licensed under the PostgreSQL License.
     HTML
 
+    version '9.6' do
+      self.release = '9.6.0'
+      self.base_url = 'https://www.postgresql.org/docs/9.6/static/'
+    end
+
     version '9.5' do
-      self.release = '9.5'
+      self.release = '9.5.4'
-      self.base_url = 'http://www.postgresql.org/docs/9.5/static/'
+      self.base_url = 'https://www.postgresql.org/docs/9.5/static/'
     end
 
     version '9.4' do
-      self.release = '9.4'
+      self.release = '9.4.9'
-      self.base_url = 'http://www.postgresql.org/docs/9.4/static/'
+      self.base_url = 'https://www.postgresql.org/docs/9.4/static/'
     end
   end
 end