diff --git a/lib/docs/filters/docker/clean_html_old.rb b/lib/docs/filters/docker/clean_html_old.rb deleted file mode 100644 index 2c572671..00000000 --- a/lib/docs/filters/docker/clean_html_old.rb +++ /dev/null @@ -1,37 +0,0 @@ -module Docs - class Docker - class CleanHtmlOldFilter < Filter - def call - if root_page? - doc.inner_html = "

Docker Documentation

" - return doc - end - - @doc = at_css('#DocumentationText') - - at_css('h2').name = 'h1' unless at_css('h1') - - css('.anchorLink', '.reading-time', 'hr', '> div[style*="margin-top"]:last-child').remove - - css('h1 + h1').each do |node| - node.name = 'h2' - end - - css('pre').each do |node| - node.content = node.content - node['data-language'] = node.parent['class'][/language-(\w+)/, 1] if node.parent['class'] - end - - css('div.highlighter-rouge').each do |node| - node.before(node.children).remove - end - - css('code.highlighter-rouge').each do |node| - node.content = node.content.gsub(/\s+/, ' ').strip - end - - doc - end - end - end -end diff --git a/lib/docs/filters/docker/clean_html_very_old.rb b/lib/docs/filters/docker/clean_html_very_old.rb deleted file mode 100644 index e0a9fedc..00000000 --- a/lib/docs/filters/docker/clean_html_very_old.rb +++ /dev/null @@ -1,22 +0,0 @@ -module Docs - class Docker - class CleanHtmlVeryOldFilter < Filter - def call - if root_page? - doc.inner_html = "

Docker Documentation

" - return doc - end - - @doc = at_css('#content') - - at_css('h2').name = 'h1' unless at_css('h1') - - css('pre').each do |node| - node.content = node.content - end - - doc - end - end - end -end diff --git a/lib/docs/filters/docker/entries.rb b/lib/docs/filters/docker/entries.rb index 63285e0c..d6a3c76f 100644 --- a/lib/docs/filters/docker/entries.rb +++ b/lib/docs/filters/docker/entries.rb @@ -10,20 +10,7 @@ module Docs def get_name return NAME_BY_SUBPATH[subpath] if NAME_BY_SUBPATH[subpath] - return at_css('h1').content unless nav_link - - name = nav_link.content.strip - name.capitalize! if name == 'exoscale' - name.remove! ' (base command)' - - if name =~ /\A[a-z\-\s]+\z/ - name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./ - name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./ - else - name << " (#{product})" if name !~ /#{product}/i && !subpath.start_with?('get-started') - end - - name + at_css('h1').content end def get_type @@ -37,11 +24,6 @@ module Docs product end - def nav_link - return @nav_link if defined?(@nav_link) - @nav_link = at_css('.currentPage') - end - def product @product ||= subpath.split('/').first.capitalize end diff --git a/lib/docs/filters/docker/entries_old.rb b/lib/docs/filters/docker/entries_old.rb deleted file mode 100644 index fdf17958..00000000 --- a/lib/docs/filters/docker/entries_old.rb +++ /dev/null @@ -1,75 +0,0 @@ -module Docs - class Docker - class EntriesOldFilter < Docs::EntriesFilter - NAME_BY_SUBPATH = { - 'engine/' => 'Engine', - 'compose/' => 'Compose', - 'machine/' => 'Machine' - } - - def get_name - return NAME_BY_SUBPATH[subpath] if NAME_BY_SUBPATH[subpath] - return at_css('h1').content unless nav_link - - name = nav_link.content.strip - name.capitalize! if name == 'exoscale' - name.remove! ' (base command)' - - if name =~ /\A[a-z\-\s]+\z/ - name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./ - name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./ - else - name << " (#{product})" if name !~ /#{product}/i - end - - name - end - - TYPE_BY_SUBPATH = { - 'engine/' => 'Engine', - 'compose/' => 'Compose', - 'machine/' => 'Machine' - } - - def get_type - return TYPE_BY_SUBPATH[subpath] if TYPE_BY_SUBPATH[subpath] - return 'Engine: CLI' if subpath.start_with?('engine/reference/commandline/') - return 'Engine: Admin Guide' if subpath.start_with?('engine/admin/') - return 'Engine: Security' if subpath.start_with?('engine/security/') - return 'Engine: Extend' if subpath.start_with?('engine/extend/') - return 'Engine: Get Started' if subpath.start_with?('engine/getstarted') - return 'Engine: Tutorials' if subpath.start_with?('engine/tutorials/') - return product if !nav_link && subpath =~ /\A\w+\/[\w\-]+\/\z/ - - leaves = nav_link.ancestors('li.leaf').reverse - return product if leaves.length <= 2 - - type = leaves[0..1].map { |node| node.at_css('> a').content.strip }.join(': ') - type.remove! %r{\ADocker } - type.remove! ' Engine' - type.sub! %r{Command[\-\s]line reference}i, 'CLI' - type.sub! 'CLI reference', 'CLI' - type - end - - def nav_link - return @nav_link if defined?(@nav_link) - @nav_link = at_css('.currentPage') - - unless @nav_link - link = at_css('#DocumentationText li a') - return unless link - link = at_css(".docsidebarnav_section a[href='#{link['href']}']") - return unless link - @nav_link = link.ancestors('.menu-closed').first.at_css('a') - end - - @nav_link - end - - def product - @product ||= subpath.split('/').first.capitalize - end - end - end -end diff --git a/lib/docs/filters/docker/entries_very_old.rb b/lib/docs/filters/docker/entries_very_old.rb deleted file mode 100644 index 25850245..00000000 --- a/lib/docs/filters/docker/entries_very_old.rb +++ /dev/null @@ -1,56 +0,0 @@ -module Docs - class Docker - class EntriesVeryOldFilter < Docs::EntriesFilter - def get_name - name = nav_link ? nav_link.content.strip : at_css('#content h1').content.strip - name.capitalize! if name == 'exoscale' - - if name =~ /\A[a-z\-\s]+\z/ - name.prepend 'docker ' if subpath =~ /engine\/reference\/commandline\/./ - name.prepend 'docker-compose ' if subpath =~ /compose\/reference\/./ - name.prepend 'docker-machine ' if subpath =~ /machine\/reference\/./ - name.prepend 'swarm ' if subpath =~ /swarm\/reference\/./ && name != 'swarm' - else - name << " (#{product})" if name !~ /#{product}/i - end - - name - end - - def get_type - unless nav_link - return 'Engine: User guide' if subpath.start_with?('engine/userguide') - end - - type = nav_link.ancestors('article').to_a.reverse.to_a[0..1].map do |node| - node.at_css('> button').content.strip - end.join(': ') - - type.remove! %r{\ADocker } - type.remove! %r{ Engine} - type.sub! %r{Command[\-\s]line reference}i, 'CLI' - type = 'Engine: Reference' if type == 'Engine: reference' - type - end - - def nav_link - return @nav_link if defined?(@nav_link) - @nav_link = at_css('#multiple .active') - - unless @nav_link - link = at_css('#content li a') - return unless link - link = at_css("#multiple a[href='#{link['href']}']") - return unless link - @nav_link = link.ancestors('article').first.at_css('button') - end - - @nav_link - end - - def product - @product ||= subpath.split('/').first.capitalize - end - end - end -end diff --git a/lib/docs/scrapers/docker.rb b/lib/docs/scrapers/docker.rb index 6328644b..ce481b3d 100644 --- a/lib/docs/scrapers/docker.rb +++ b/lib/docs/scrapers/docker.rb @@ -1,11 +1,31 @@ module Docs class Docker < UrlScraper + include MultipleBaseUrls + + self.name = 'Docker' self.type = 'simple' + self.release = '20.10.6' + + self.base_urls = [ + 'https://docs.docker.com/', + 'https://docs.docker.com/machine/' + ] + self.links = { home: 'https://docker.com/', code: 'https://github.com/docker/docker' } + html_filters.push 'docker/entries', 'docker/clean_html' + + options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//] + + options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/] + + options[:skip] = [ + 'engine/userguide/networking/get-started-overlay/' + ] + options[:trailing_slash] = true options[:fix_urls] = ->(url) do @@ -21,242 +41,54 @@ module Docs Docker, Inc. and other parties may also have trademark rights in other terms used herein. HTML + options[:replace_paths] = { + 'engine/userguide/' => 'config/daemon', + 'engine/userguide/labels-custom-metadata/' => 'config', + 'engine/swarm/networking/' => 'network/overlay', + 'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images/dockerfile_best-practices/', + 'engine/userguide/networking/get-started-overlay/' => 'network/network-tutorial-overlay/', + 'engine/userguide/networking/' => 'network/', + 'engine/reference/api/' => 'develop/sdk', + 'engine/api/latest/' => 'develop/sdk', + 'get-started/part3/' => 'get-started/04_sharing_app/', + 'engine/security/https/' => 'engine/security/protect-access/', + 'compose/aspnet-mssql-compose/' => 'samples/aspnet-mssql-compose/', + 'engine/examples/dotnetcore/' => 'samples/dotnetcore/' + } + version '19' do self.release = '19.03' - self.base_url = "https://docs.docker.com/" - - html_filters.push 'docker/entries', 'docker/clean_html' - - options[:container] = '.wrapper .container-fluid .row' - - options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//] - options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/] - - options[:replace_paths] = { - 'install/linux/ubuntu/' => 'install/linux/docker-ce', - 'get-started/part1' => 'get-started', - 'engine/installation/' => 'install', - 'engine/installation/linux/linux-postinstall/' => 'install/linux', - 'compose/overview/' => 'compose', - 'docker-cloud/' => 'docker-hub', - 'datacenter/install/linux/' => 'ee', - 'engine/userguide/' => 'config/daemon', - 'engine/admin/' => 'config/daemon', - 'opensource/get-help/' => 'opensource', - 'engine/tutorials/dockerimages/' => 'get-started', - 'engine/admin/volumes/bind-mounts/' => 'storage', - 'engine/tutorials/dockervolumes/' => 'storage', - 'engine/admin/volumes/volumes/' => 'storage', - 'engine/userguide/labels-custom-metadata/' => 'config', - 'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images', - 'engine/swarm/networking/' => 'network', - 'engine/admin/resource_constraints/' => 'config/containers', - 'engine/admin/logging/overview/' => 'config/containers/logging', - 'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images', - 'engine/tutorials/dockerrepos/' => 'get-started', - 'engine/userguide/networking/' => 'network', - 'engine/userguide/networking/get-started-overlay/' => 'network', - 'engine/reference/commandline/swarm_join_token/' => 'engine/reference/commandline', - 'engine/understanding-docker/' => 'engine', - 'engine/userguide/dockervolumes/' => 'storage', - 'engine/installation/binaries/' => 'install/linux/docker-ce', - 'engine/userguide/networking/default_network/dockerlinks/' => 'network', - 'engine/reference/api/' => 'develop/sdk', - 'engine/admin/systemd/' => 'config/daemon', - 'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver', - 'engine/api/' => 'develop/sdk', - 'engine/userguide/networking/get-started-overlay' => 'network', - 'engine/userguide/networking/overlay-security-model/' => 'network', - 'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce', - 'engine/admin/volumes/' => 'storage/volumes/', - 'engine/userguide/networking//' => 'network', - 'engine/reference/commandline' => 'engine/reference/commandline/docker', - 'engine/reference/commandline/' => 'engine/reference/commandline/docker/', - } + self.base_url = "https://docs.docker.com" end version '18' do self.release = '18.09' - self.base_url = "https://docs.docker.com/v#{release}/" - - html_filters.push 'docker/entries', 'docker/clean_html' - - options[:container] = '.wrapper .container-fluid .row' - - options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//] - options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/] - - options[:replace_paths] = { - 'install/linux/ubuntu/' => 'install/linux/docker-ce', - 'get-started/part1' => 'get-started', - 'engine/installation/' => 'install', - 'engine/installation/linux/linux-postinstall/' => 'install/linux', - 'compose/overview/' => 'compose', - 'datacenter/install/linux/' => 'ee', - 'engine/userguide/' => 'config/daemon', - 'engine/admin/' => 'config/daemon', - 'opensource/get-help/' => 'opensource', - 'engine/tutorials/dockerimages/' => 'get-started', - 'engine/admin/volumes/bind-mounts/' => 'storage', - 'engine/tutorials/dockervolumes/' => 'storage', - 'engine/admin/volumes/volumes/' => 'storage', - 'engine/userguide/labels-custom-metadata/' => 'config', - 'engine/reference/api/' => 'develop/sdk', - 'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images', - 'engine/swarm/networking/' => 'network', - 'engine/admin/resource_constraints/' => 'config/containers', - 'engine/admin/logging/overview/' => 'config/containers/logging', - 'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images', - 'engine/tutorials/dockerrepos/' => 'get-started', - 'engine/userguide/networking/' => 'network', - 'engine/userguide/networking/get-started-overlay/' => 'network', - 'engine/understanding-docker/' => 'engine', - 'engine/reference/commandline/swarm_join_token/' => 'engine/reference/commandline', - 'engine/userguide/dockervolumes/' => 'storage', - 'engine/admin/systemd/' => 'config/daemon', - 'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver', - 'engine/installation/binaries/' => 'install/linux/docker-ce', - 'engine/userguide/networking/default_network/dockerlinks/' => 'network', - 'engine/userguide/networking/overlay-security-model/' => 'network', - 'engine/userguide/networking/get-started-overlay' => 'network', - 'engine/api/' => 'develop/sdk', - 'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce', - 'engine/admin/volumes/' => 'storage/volumes/', - 'engine/userguide/networking//' => 'network', - 'engine/reference/commandline' => 'engine/reference/commandline/docker', - 'engine/reference/commandline/' => 'engine/reference/commandline/docker/', - } + self.base_url = "https://docs.docker.com" end version '17' do self.release = '17.12' - self.base_url = "https://docs.docker.com/v#{release}/" - - html_filters.push 'docker/entries', 'docker/clean_html' - - options[:container] = '.wrapper .container-fluid .row' - - options[:only_patterns] = [/\Aget-started\//, /\Aengine\//, /\Acompose\//, /\Amachine\//, /\Anotary\//] - options[:skip_patterns] = [/\Aengine\/api\/v/, /glossary/, /docker-ee/] - - options[:replace_paths] = { - 'get-started/part1' => 'get-started', - 'engine/installation/' => 'install', - 'engine/installation/linux/linux-postinstall/' => 'install/linux', - 'opensource/get-help/' => 'opensource', - 'engine/admin/volumes/volumes/' => 'storage', - 'engine/tutorials/dockerimages/' => 'get-started', - 'engine/admin/volumes/bind-mounts/' => 'storage', - 'engine/tutorials/dockervolumes/' => 'storage', - 'datacenter/install/aws/' => 'docker-for-aws', - 'engine/userguide/' => 'config/daemon', - 'engine/admin/' => 'config/daemon', - 'engine/userguide/labels-custom-metadata/' => 'config', - 'engine/userguide/eng-image/multistage-build/' => 'develop/develop-images', - 'engine/swarm/networking/' => 'network', - 'engine/admin/resource_constraints/' => 'config/containers', - 'engine/admin/logging/overview/' => 'config/containers/logging', - 'engine/understanding-docker/' => 'engine', - 'engine/userguide/eng-image/dockerfile_best-practices/' => 'develop/develop-images', - 'engine/tutorials/dockerrepos/' => 'get-started', - 'engine/userguide/networking/' => 'network', - 'engine/reference/commandline/swarm_join_token/' => 'edge/engine/reference/commandline', - 'engine/userguide/networking/get-started-overlay/' => 'network', - 'engine/userguide/dockervolumes/' => 'storage', - 'engine/installation/binaries/' => 'install/linux/docker-ce', - 'engine/userguide/networking/default_network/dockerlinks/' => 'network', - 'engine/reference/api/' => 'develop/sdk', - 'engine/admin/live-restore/' => 'config/containers', - 'engine/api/' => 'develop/sdk', - 'engine/userguide/networking/get-started-overlay' => 'network', - 'security/security/' => 'engine/security', - 'engine/installation/linux/docker-ce/binaries/' => 'install/linux/docker-ce', - 'engine/reference/commandline/' => 'edge/engine/reference/commandline', - 'engine/admin/systemd/' => 'config/daemon', - 'engine/userguide/storagedriver/imagesandcontainers/' => 'storage/storagedriver', - 'engine/userguide/networking/overlay-security-model/' => 'network', - 'engine/admin/volumes/' => 'storage/volumes/', - 'engine/userguide/networking//' => 'network', - } - end - - module OldOptions - def self.included(klass) - klass.options[:only_patterns] = [/\Aengine\//, /\Acompose\//, /\Amachine\//] - klass.options[:skip_patterns] = [/\Aengine\/api\/v/, /\Aengine\/installation/] - klass.options[:skip] = %w( - / - engine/userguide/ - engine/examples/ - engine/reference/ - engine/reference/api/ - engine/reference/api/docker_remote_api_v1.24/ - engine/getstarted/linux_install_help/ - machine/reference/ - machine/drivers/ - machine/examples/ - compose/reference/ - ) - klass.options[:replace_paths] = { - 'engine/userguide/networking/dockernetworks/' => 'engine/userguide/networking/', - 'engine/userguide/dockervolumes/' => 'engine/tutorials/dockervolumes/', - 'engine/reference/logging/overview/' => 'engine/admin/logging/overview/', - 'engine/reference/commandline/daemon/' => 'engine/reference/commandline/dockerd/', - 'engine/reference/commandline/' => 'engine/reference/commandline/docker/', - 'engine/reference/api/docker_remote_api/' => 'engine/api/', - 'engine/swarm/how-swarm-mode-works/' => 'engine/swarm/how-swarm-mode-works/nodes/', - 'engine/tutorials/dockerizing/' => 'engine/getstarted/step_one/', - 'engine/tutorials/usingdocker/' => 'engine/getstarted/step_three/', - 'engine/tutorials/dockerimages/' => 'engine/getstarted/step_four/', - 'engine/tutorials/dockerrepos/' => 'engine/getstarted/step_six/' - } - end + self.base_url = "https://docs.docker.com" end version '1.13' do - include OldOptions - self.release = '1.13' - self.base_url = "https://docs.docker.com/v#{self.version}/" - - html_filters.push 'docker/entries_old', 'docker/clean_html_old' - - options[:container] = '.container-fluid .row' + self.base_url = "https://docs.docker.com" end version '1.12' do - include OldOptions - self.release = '1.12' - self.base_url = "https://docs.docker.com/v#{self.version}/" - - html_filters.push 'docker/entries_old', 'docker/clean_html_old' - - options[:container] = '.container-fluid .row' + self.base_url = "https://docs.docker.com" end version '1.11' do - include OldOptions - self.release = '1.11' - self.base_url = "https://docs.docker.com/v#{self.version}/" - - html_filters.push 'docker/entries_very_old', 'docker/clean_html_very_old' - - options[:container] = '#docs' - options[:only_patterns] << /\Aswarm\// + self.base_url = "https://docs.docker.com" end version '1.10' do - include OldOptions - self.release = '1.10' - self.base_url = "https://docs.docker.com/v#{self.version}/" - - html_filters.push 'docker/entries_very_old', 'docker/clean_html_very_old' - - options[:container] = '#docs' - options[:only_patterns] << /\Aswarm\// + self.base_url = "https://docs.docker.com" end def get_latest_version(opts)