From 0ad6fefa83e4818ac1a170239acb1d6a3e690005 Mon Sep 17 00:00:00 2001 From: Enoc Date: Tue, 20 Apr 2021 01:46:30 -0600 Subject: [PATCH 1/5] Add support for compatibility tables almost all the mdn docs works, but some have minor issues. --- lib/docs/core/compat_tables.rb | 184 ++++++++++++++++++++++ lib/docs/core/filter.rb | 5 + lib/docs/filters/css/clean_html.rb | 14 ++ lib/docs/filters/dom/clean_html.rb | 14 ++ lib/docs/filters/html/clean_html.rb | 12 ++ lib/docs/filters/javascript/clean_html.rb | 13 ++ lib/docs/filters/svg/clean_html.rb | 15 +- lib/docs/filters/xslt_xpath/clean_html.rb | 13 ++ 8 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 lib/docs/core/compat_tables.rb diff --git a/lib/docs/core/compat_tables.rb b/lib/docs/core/compat_tables.rb new file mode 100644 index 00000000..fcc1abeb --- /dev/null +++ b/lib/docs/core/compat_tables.rb @@ -0,0 +1,184 @@ +module CompatTables + require 'json' + require 'net/http' + require 'pry' + + BROWSERS = { + 'chrome' => 'Chrome', + 'edge' => 'Edge', + 'firefox' => 'Firefox', + 'ie' => 'Internet Explorer', + 'opera' => 'Opera', + 'safari' => 'Safari', + 'webview_android' => 'WebView Android', + 'chrome_android' => 'Chrome Android', + 'firefox_android' => 'Firefox for Android', + 'opera_android' => 'Opera Android', + 'safari_ios' => 'Safari on IOS', + 'samsunginternet_android' => 'Samsung Internet' + } + + def generate_compatibility_table() + json_files_uri = request_bcd_uris() + + compat_tables = [] + + json_files_uri.each do |uri| + compat_tables.push(generate_compatibility_table_wrapper(uri)) + end + + return compat_tables + end + + def request_bcd_uris + index_json = JSON.load(Net::HTTP.get(URI(current_url.to_s + '/index.json'))) + + uris = [] + + index_json['doc']['body'].each do |element| + uris.push(element['value']['dataURL']) if element['type'] == 'browser_compatibility' + end + + uris.map! do |uri| + tmp_uri = URI.parse(base_url.to_s) + tmp_uri.path = uri + uri = tmp_uri.to_s + end + + return uris + end + + def generate_compatibility_table_wrapper(uri) + + @json_data = JSON.load(Net::HTTP.get(URI(uri)))['data'] + + html_table = generate_basic_html_table() + + @json_data.keys.each do |key| + if key == '__compat' or @json_data[key]['__compat'] + add_entry_to_table(html_table, key) + else + end + end + + return html_table + end + + def generate_basic_html_table + table = Nokogiri::XML::Node.new('table', doc) + + table.add_child('') + + table.css('#bct-browser-type').each do |node| + node.add_child('') + %w(Desktop Mobile).each do |browser_type| + node.add_child("#{browser_type}") + end + end + + table.css('#bct-browsers').each do |node| + node.add_child('') + + BROWSERS.values.each do |browser| + node.add_child("#{browser}") + end + end + + return table + end + + def add_entry_to_table(html_table, key) + json = @json_data[key] + + html_table.at_css('tbody').add_child('') + + last_table_entry = html_table.at_css('tbody').last_element_child + + last_table_entry.add_child("#{key}") + + BROWSERS.keys.each do |browser_key| + if key == '__compat' + add_data_to_entry(json['support'][browser_key], last_table_entry) + else + add_data_to_entry(json['__compat']['support'][browser_key], last_table_entry) + end + + end + end + + def add_data_to_entry(json, entry) + version_added = [] + version_removed = [] + notes = [] + + if json.is_a?(Array) + json.each do |element| + + if element['version_added'] + version_added.push(element['version_added']) + else + version_added.push(false) + end + + if element['version_removed'] + version_removed.push(element['version_removed']) + else + version_removed.push(false) + end + + if element['notes'] + notes.push(element['notes']) + else + notes.push(false) + end + + end + else + version_added.push(json['version_added']) + version_removed.push(json['version_removed']) + notes.push(json['notes']) + end + + version_added.map! do |version| + if version == true + version = 'Yes' + elsif version == false + version = 'No' + elsif version.is_a?(Numeric) + else + version = '?' + end + + version + end + + if version_removed[0] + format_string = "" + else + if version_added[0] == 'No' + format_string = "" + else + format_string = "" + end + end + + for value in (0..version_added.length-1) do + if version_removed[value] + format_string += "
#{version_added[value]}-#{version_removed[value]}
" + else + if version_added[value] == 'No' + format_string += "
#{version_added[value]}
" + else + format_string += "
#{version_added[value]}
" + end + end + + if notes[value] + format_string += "
#{notes[value]}
" + end + end + + entry.add_child(format_string) + end + +end diff --git a/lib/docs/core/filter.rb b/lib/docs/core/filter.rb index 5be77883..8f6bf173 100644 --- a/lib/docs/core/filter.rb +++ b/lib/docs/core/filter.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true +require './lib/docs/core/compat_tables.rb' + module Docs class Filter < ::HTML::Pipeline::Filter + + include CompatTables + def css(*args) doc.css(*args) end diff --git a/lib/docs/filters/css/clean_html.rb b/lib/docs/filters/css/clean_html.rb index f202f7c9..a081e32a 100644 --- a/lib/docs/filters/css/clean_html.rb +++ b/lib/docs/filters/css/clean_html.rb @@ -26,7 +26,21 @@ module Docs css('img[style*="float"]').each do |node| node['style'] = node['style'] + ';float: none; display: block;' end + + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + end + end end end diff --git a/lib/docs/filters/dom/clean_html.rb b/lib/docs/filters/dom/clean_html.rb index d6704468..d7c3981c 100644 --- a/lib/docs/filters/dom/clean_html.rb +++ b/lib/docs/filters/dom/clean_html.rb @@ -38,7 +38,21 @@ module Docs node['class'] = 'syntaxbox' node.parent.before(node).remove end + + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + end + end end end diff --git a/lib/docs/filters/html/clean_html.rb b/lib/docs/filters/html/clean_html.rb index 0d6b43b4..ee090ef2 100644 --- a/lib/docs/filters/html/clean_html.rb +++ b/lib/docs/filters/html/clean_html.rb @@ -6,6 +6,18 @@ module Docs node.before(node.children).remove end + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + doc end end diff --git a/lib/docs/filters/javascript/clean_html.rb b/lib/docs/filters/javascript/clean_html.rb index eeb23ba8..0187e4bf 100644 --- a/lib/docs/filters/javascript/clean_html.rb +++ b/lib/docs/filters/javascript/clean_html.rb @@ -19,6 +19,19 @@ module Docs css('div > .overheadIndicator:first-child:last-child', 'div > .blockIndicator:first-child:last-child').each do |node| node.parent.replace(node) end + + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + end end end diff --git a/lib/docs/filters/svg/clean_html.rb b/lib/docs/filters/svg/clean_html.rb index 3468cfb7..15904229 100644 --- a/lib/docs/filters/svg/clean_html.rb +++ b/lib/docs/filters/svg/clean_html.rb @@ -7,7 +7,7 @@ module Docs end def root - doc.inner_html = doc.at_css('#Documentation + dl').to_html + end def other @@ -26,7 +26,20 @@ module Docs node['class'] = 'index' css('h3').each { |n| n.name = 'span' } end + + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end end + end end end diff --git a/lib/docs/filters/xslt_xpath/clean_html.rb b/lib/docs/filters/xslt_xpath/clean_html.rb index 82096b20..e7c62084 100644 --- a/lib/docs/filters/xslt_xpath/clean_html.rb +++ b/lib/docs/filters/xslt_xpath/clean_html.rb @@ -24,6 +24,19 @@ module Docs child = child.next while child && child.text? && child.content.blank? child.remove if child.try(:name) == 'br' end + + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + end end end From 5b3e3b3e531aac3863824bb4688626096f1a3692 Mon Sep 17 00:00:00 2001 From: Enoc Date: Mon, 3 May 2021 01:35:12 -0600 Subject: [PATCH 2/5] Fix minor issues of the compatibility tables --- lib/docs/core/compat_tables.rb | 112 ++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/lib/docs/core/compat_tables.rb b/lib/docs/core/compat_tables.rb index fcc1abeb..82fbac31 100644 --- a/lib/docs/core/compat_tables.rb +++ b/lib/docs/core/compat_tables.rb @@ -1,7 +1,6 @@ module CompatTables require 'json' require 'net/http' - require 'pry' BROWSERS = { 'chrome' => 'Chrome', @@ -36,7 +35,7 @@ module CompatTables uris = [] index_json['doc']['body'].each do |element| - uris.push(element['value']['dataURL']) if element['type'] == 'browser_compatibility' + uris.push(element['value']['dataURL']) if element['type'] == 'browser_compatibility' and element['value']['dataURL'] end uris.map! do |uri| @@ -94,7 +93,13 @@ module CompatTables last_table_entry = html_table.at_css('tbody').last_element_child - last_table_entry.add_child("#{key}") + if key == '__compat' + tmp = slug.split('/') + last_table_entry.add_child("#{tmp.last}") + else + last_table_entry.add_child("#{key}") + end + BROWSERS.keys.each do |browser_key| if key == '__compat' @@ -111,71 +116,76 @@ module CompatTables version_removed = [] notes = [] - if json.is_a?(Array) - json.each do |element| + if json + if json.is_a?(Array) + json.each do |element| - if element['version_added'] - version_added.push(element['version_added']) - else - version_added.push(false) - end + if element['version_added'] + version_added.push(element['version_added']) + else + version_added.push(false) + end + + if element['version_removed'] + version_removed.push(element['version_removed']) + else + version_removed.push(false) + end + + if element['notes'] + notes.push(element['notes']) + else + notes.push(false) + end - if element['version_removed'] - version_removed.push(element['version_removed']) - else - version_removed.push(false) end + else + version_added.push(json['version_added']) + version_removed.push(json['version_removed']) + notes.push(json['notes']) + end - if element['notes'] - notes.push(element['notes']) + version_added.map! do |version| + if version == true + version = 'Yes' + elsif version == false + version = 'No' + elsif version.is_a?(Numeric) else - notes.push(false) + version = '?' end + version end - else - version_added.push(json['version_added']) - version_removed.push(json['version_removed']) - notes.push(json['notes']) - end - - version_added.map! do |version| - if version == true - version = 'Yes' - elsif version == false - version = 'No' - elsif version.is_a?(Numeric) - else - version = '?' - end - - version - end - if version_removed[0] - format_string = "" - else - if version_added[0] == 'No' + if version_removed[0] format_string = "" else - format_string = "" + if version_added[0] == 'No' + format_string = "" + else + format_string = "" + end end - end - for value in (0..version_added.length-1) do - if version_removed[value] - format_string += "
#{version_added[value]}-#{version_removed[value]}
" - else - if version_added[value] == 'No' - format_string += "
#{version_added[value]}
" + for value in (0..version_added.length-1) do + if version_removed[value] + format_string += "
#{version_added[value]}-#{version_removed[value]}
" else - format_string += "
#{version_added[value]}
" + if version_added[value] == 'No' + format_string += "
#{version_added[value]}
" + else + format_string += "
#{version_added[value]}
" + end end - end - if notes[value] - format_string += "
#{notes[value]}
" + if notes[value] + format_string += "
#{notes[value]}
" + end end + + else + format_string = "
?
" end entry.add_child(format_string) From 9a297edc84a67b141d400238948202d0b55234df Mon Sep 17 00:00:00 2001 From: Enoc Date: Tue, 4 May 2021 20:59:43 -0600 Subject: [PATCH 3/5] Convert compat tables module to a filter. - Fix numbers in tables. --- lib/docs/core/compat_tables.rb | 194 -------------------- lib/docs/core/filter.rb | 5 - lib/docs/filters/css/clean_html.rb | 14 -- lib/docs/filters/dom/clean_html.rb | 14 -- lib/docs/filters/html/clean_html.rb | 12 -- lib/docs/filters/javascript/clean_html.rb | 13 -- lib/docs/filters/mdn/compat_tables.rb | 212 ++++++++++++++++++++++ lib/docs/filters/svg/clean_html.rb | 14 -- lib/docs/filters/xslt_xpath/clean_html.rb | 13 -- lib/docs/scrapers/mdn/mdn.rb | 2 +- 10 files changed, 213 insertions(+), 280 deletions(-) delete mode 100644 lib/docs/core/compat_tables.rb create mode 100644 lib/docs/filters/mdn/compat_tables.rb diff --git a/lib/docs/core/compat_tables.rb b/lib/docs/core/compat_tables.rb deleted file mode 100644 index 82fbac31..00000000 --- a/lib/docs/core/compat_tables.rb +++ /dev/null @@ -1,194 +0,0 @@ -module CompatTables - require 'json' - require 'net/http' - - BROWSERS = { - 'chrome' => 'Chrome', - 'edge' => 'Edge', - 'firefox' => 'Firefox', - 'ie' => 'Internet Explorer', - 'opera' => 'Opera', - 'safari' => 'Safari', - 'webview_android' => 'WebView Android', - 'chrome_android' => 'Chrome Android', - 'firefox_android' => 'Firefox for Android', - 'opera_android' => 'Opera Android', - 'safari_ios' => 'Safari on IOS', - 'samsunginternet_android' => 'Samsung Internet' - } - - def generate_compatibility_table() - json_files_uri = request_bcd_uris() - - compat_tables = [] - - json_files_uri.each do |uri| - compat_tables.push(generate_compatibility_table_wrapper(uri)) - end - - return compat_tables - end - - def request_bcd_uris - index_json = JSON.load(Net::HTTP.get(URI(current_url.to_s + '/index.json'))) - - uris = [] - - index_json['doc']['body'].each do |element| - uris.push(element['value']['dataURL']) if element['type'] == 'browser_compatibility' and element['value']['dataURL'] - end - - uris.map! do |uri| - tmp_uri = URI.parse(base_url.to_s) - tmp_uri.path = uri - uri = tmp_uri.to_s - end - - return uris - end - - def generate_compatibility_table_wrapper(uri) - - @json_data = JSON.load(Net::HTTP.get(URI(uri)))['data'] - - html_table = generate_basic_html_table() - - @json_data.keys.each do |key| - if key == '__compat' or @json_data[key]['__compat'] - add_entry_to_table(html_table, key) - else - end - end - - return html_table - end - - def generate_basic_html_table - table = Nokogiri::XML::Node.new('table', doc) - - table.add_child('') - - table.css('#bct-browser-type').each do |node| - node.add_child('') - %w(Desktop Mobile).each do |browser_type| - node.add_child("#{browser_type}") - end - end - - table.css('#bct-browsers').each do |node| - node.add_child('') - - BROWSERS.values.each do |browser| - node.add_child("#{browser}") - end - end - - return table - end - - def add_entry_to_table(html_table, key) - json = @json_data[key] - - html_table.at_css('tbody').add_child('') - - last_table_entry = html_table.at_css('tbody').last_element_child - - if key == '__compat' - tmp = slug.split('/') - last_table_entry.add_child("#{tmp.last}") - else - last_table_entry.add_child("#{key}") - end - - - BROWSERS.keys.each do |browser_key| - if key == '__compat' - add_data_to_entry(json['support'][browser_key], last_table_entry) - else - add_data_to_entry(json['__compat']['support'][browser_key], last_table_entry) - end - - end - end - - def add_data_to_entry(json, entry) - version_added = [] - version_removed = [] - notes = [] - - if json - if json.is_a?(Array) - json.each do |element| - - if element['version_added'] - version_added.push(element['version_added']) - else - version_added.push(false) - end - - if element['version_removed'] - version_removed.push(element['version_removed']) - else - version_removed.push(false) - end - - if element['notes'] - notes.push(element['notes']) - else - notes.push(false) - end - - end - else - version_added.push(json['version_added']) - version_removed.push(json['version_removed']) - notes.push(json['notes']) - end - - version_added.map! do |version| - if version == true - version = 'Yes' - elsif version == false - version = 'No' - elsif version.is_a?(Numeric) - else - version = '?' - end - - version - end - - if version_removed[0] - format_string = "" - else - if version_added[0] == 'No' - format_string = "" - else - format_string = "" - end - end - - for value in (0..version_added.length-1) do - if version_removed[value] - format_string += "
#{version_added[value]}-#{version_removed[value]}
" - else - if version_added[value] == 'No' - format_string += "
#{version_added[value]}
" - else - format_string += "
#{version_added[value]}
" - end - end - - if notes[value] - format_string += "
#{notes[value]}
" - end - end - - else - format_string = "
?
" - end - - entry.add_child(format_string) - end - -end diff --git a/lib/docs/core/filter.rb b/lib/docs/core/filter.rb index 8f6bf173..5be77883 100644 --- a/lib/docs/core/filter.rb +++ b/lib/docs/core/filter.rb @@ -1,12 +1,7 @@ # frozen_string_literal: true -require './lib/docs/core/compat_tables.rb' - module Docs class Filter < ::HTML::Pipeline::Filter - - include CompatTables - def css(*args) doc.css(*args) end diff --git a/lib/docs/filters/css/clean_html.rb b/lib/docs/filters/css/clean_html.rb index a081e32a..f202f7c9 100644 --- a/lib/docs/filters/css/clean_html.rb +++ b/lib/docs/filters/css/clean_html.rb @@ -26,21 +26,7 @@ module Docs css('img[style*="float"]').each do |node| node['style'] = node['style'] + ';float: none; display: block;' end - - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end - end - end end end diff --git a/lib/docs/filters/dom/clean_html.rb b/lib/docs/filters/dom/clean_html.rb index d7c3981c..d6704468 100644 --- a/lib/docs/filters/dom/clean_html.rb +++ b/lib/docs/filters/dom/clean_html.rb @@ -38,21 +38,7 @@ module Docs node['class'] = 'syntaxbox' node.parent.before(node).remove end - - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end - end - end end end diff --git a/lib/docs/filters/html/clean_html.rb b/lib/docs/filters/html/clean_html.rb index ee090ef2..0d6b43b4 100644 --- a/lib/docs/filters/html/clean_html.rb +++ b/lib/docs/filters/html/clean_html.rb @@ -6,18 +6,6 @@ module Docs node.before(node.children).remove end - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end - doc end end diff --git a/lib/docs/filters/javascript/clean_html.rb b/lib/docs/filters/javascript/clean_html.rb index 0187e4bf..eeb23ba8 100644 --- a/lib/docs/filters/javascript/clean_html.rb +++ b/lib/docs/filters/javascript/clean_html.rb @@ -19,19 +19,6 @@ module Docs css('div > .overheadIndicator:first-child:last-child', 'div > .blockIndicator:first-child:last-child').each do |node| node.parent.replace(node) end - - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end - end end end diff --git a/lib/docs/filters/mdn/compat_tables.rb b/lib/docs/filters/mdn/compat_tables.rb new file mode 100644 index 00000000..8f81b488 --- /dev/null +++ b/lib/docs/filters/mdn/compat_tables.rb @@ -0,0 +1,212 @@ +module Docs + class Mdn + class CompatTablesFilter < Filter + + def call + if at_css('#browser_compatibility') \ + and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ + and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') + + at_css('#browser_compatibility').next_sibling.remove + + compatibility_tables = generate_compatibility_table() + compatibility_tables.each do |table| + at_css('#browser_compatibility').add_next_sibling(table) + end + end + + doc + end + + BROWSERS = { + 'chrome' => 'Chrome', + 'edge' => 'Edge', + 'firefox' => 'Firefox', + 'ie' => 'Internet Explorer', + 'opera' => 'Opera', + 'safari' => 'Safari', + 'webview_android' => 'WebView Android', + 'chrome_android' => 'Chrome Android', + 'firefox_android' => 'Firefox for Android', + 'opera_android' => 'Opera Android', + 'safari_ios' => 'Safari on IOS', + 'samsunginternet_android' => 'Samsung Internet' + } + + def generate_compatibility_table() + json_files_uri = request_bcd_uris() + + compat_tables = [] + + json_files_uri.each do |uri| + compat_tables.push(generate_compatibility_table_wrapper(uri)) + end + + return compat_tables + end + + def request_bcd_uris + index_json = JSON.load(Net::HTTP.get(URI(current_url.to_s + '/index.json'))) + + uris = [] + + index_json['doc']['body'].each do |element| + uris.push(element['value']['dataURL']) if element['type'] == 'browser_compatibility' and element['value']['dataURL'] + end + + uris.map! do |uri| + tmp_uri = URI.parse(base_url.to_s) + tmp_uri.path = uri + uri = tmp_uri.to_s + end + + return uris + end + + def generate_compatibility_table_wrapper(uri) + + @json_data = JSON.load(Net::HTTP.get(URI(uri)))['data'] + + html_table = generate_basic_html_table() + + @json_data.keys.each do |key| + if key == '__compat' or @json_data[key]['__compat'] + add_entry_to_table(html_table, key) + else + end + end + + return html_table + end + + def generate_basic_html_table + table = Nokogiri::XML::Node.new('table', doc) + + table.add_child('') + + table.css('#bct-browser-type').each do |node| + node.add_child('') + %w(Desktop Mobile).each do |browser_type| + node.add_child("#{browser_type}") + end + end + + table.css('#bct-browsers').each do |node| + node.add_child('') + + BROWSERS.values.each do |browser| + node.add_child("#{browser}") + end + end + + return table + end + + def add_entry_to_table(html_table, key) + json = @json_data[key] + + html_table.at_css('tbody').add_child('') + + last_table_entry = html_table.at_css('tbody').last_element_child + + if key == '__compat' + tmp = slug.split('/') + last_table_entry.add_child("#{tmp.last}") + else + last_table_entry.add_child("#{key}") + end + + + BROWSERS.keys.each do |browser_key| + if key == '__compat' + add_data_to_entry(json['support'][browser_key], last_table_entry) + else + add_data_to_entry(json['__compat']['support'][browser_key], last_table_entry) + end + + end + end + + def add_data_to_entry(json, entry) + version_added = [] + version_removed = [] + notes = [] + + if json + if json.is_a?(Array) + json.each do |element| + + if element['version_added'] + version_added.push(element['version_added']) + else + version_added.push(false) + end + + if element['version_removed'] + version_removed.push(element['version_removed']) + else + version_removed.push(false) + end + + if element['notes'] + notes.push(element['notes']) + else + notes.push(false) + end + + end + else + version_added.push(json['version_added']) + version_removed.push(json['version_removed']) + notes.push(json['notes']) + end + + version_added.map! do |version| + if version == true + version = 'Yes' + elsif version == false + version = 'No' + elsif version.is_a?(String) + else + version = '?' + end + + version + end + + if version_removed[0] + format_string = "" + else + if version_added[0] == 'No' + format_string = "" + else + format_string = "" + end + end + + for value in (0..version_added.length-1) do + if version_removed[value] + format_string += "
#{version_added[value]}-#{version_removed[value]}
" + else + if version_added[value] == 'No' + format_string += "
#{version_added[value]}
" + else + format_string += "
#{version_added[value]}
" + end + end + + if notes[value] + format_string += "
#{notes[value]}
" + end + end + + else + format_string = "
?
" + end + + entry.add_child(format_string) + end + + end + end +end diff --git a/lib/docs/filters/svg/clean_html.rb b/lib/docs/filters/svg/clean_html.rb index 15904229..fa9003f3 100644 --- a/lib/docs/filters/svg/clean_html.rb +++ b/lib/docs/filters/svg/clean_html.rb @@ -7,7 +7,6 @@ module Docs end def root - end def other @@ -26,20 +25,7 @@ module Docs node['class'] = 'index' css('h3').each { |n| n.name = 'span' } end - - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end end - end end end diff --git a/lib/docs/filters/xslt_xpath/clean_html.rb b/lib/docs/filters/xslt_xpath/clean_html.rb index e7c62084..82096b20 100644 --- a/lib/docs/filters/xslt_xpath/clean_html.rb +++ b/lib/docs/filters/xslt_xpath/clean_html.rb @@ -24,19 +24,6 @@ module Docs child = child.next while child && child.text? && child.content.blank? child.remove if child.try(:name) == 'br' end - - if at_css('#browser_compatibility') \ - and not at_css('#browser_compatibility').next_sibling.classes.include?('warning') \ - and not at_css('#browser_compatibility').next_sibling.content.match?('Supported') - - at_css('#browser_compatibility').next_sibling.remove - - compatibility_tables = generate_compatibility_table() - compatibility_tables.each do |table| - at_css('#browser_compatibility').add_next_sibling(table) - end - end - end end end diff --git a/lib/docs/scrapers/mdn/mdn.rb b/lib/docs/scrapers/mdn/mdn.rb index 03a1d44b..2b31668a 100644 --- a/lib/docs/scrapers/mdn/mdn.rb +++ b/lib/docs/scrapers/mdn/mdn.rb @@ -3,7 +3,7 @@ module Docs self.abstract = true self.type = 'mdn' - html_filters.push 'mdn/clean_html' + html_filters.push 'mdn/clean_html', 'mdn/compat_tables' options[:container] = '#content > .main-page-content' options[:trailing_slash] = false From 3cf987da8ba70d252af6384df283fbdf0f631212 Mon Sep 17 00:00:00 2001 From: Enoc Date: Wed, 5 May 2021 21:43:58 -0600 Subject: [PATCH 4/5] Fix mdn scrapers - xslt_xpath had an error with redirection. --- lib/docs/filters/xslt_xpath/entries.rb | 5 +++-- lib/docs/scrapers/mdn/css.rb | 2 +- lib/docs/scrapers/mdn/dom.rb | 2 +- lib/docs/scrapers/mdn/javascript.rb | 2 +- lib/docs/scrapers/mdn/svg.rb | 2 +- lib/docs/scrapers/mdn/xslt_xpath.rb | 6 +----- 6 files changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/docs/filters/xslt_xpath/entries.rb b/lib/docs/filters/xslt_xpath/entries.rb index 5151246f..3480547f 100644 --- a/lib/docs/filters/xslt_xpath/entries.rb +++ b/lib/docs/filters/xslt_xpath/entries.rb @@ -6,13 +6,14 @@ module Docs name.remove! 'XPath.' name.remove! 'XSLT.' name.remove! 'Axes.' - name.prepend 'xsl:' if slug =~ /\AXSLT\/[a-z]/ + name.remove! 'Element.' + name.prepend 'xsl:' if slug =~ /XSLT\/Element/ name << '()' if name.gsub!('Functions.', '') name end def get_type - if slug =~ /\AXSLT\/[a-z]/ + if slug =~ /XSLT\/Element/ 'XSLT Elements' elsif slug.start_with?('XPath/Axes') 'XPath Axes' diff --git a/lib/docs/scrapers/mdn/css.rb b/lib/docs/scrapers/mdn/css.rb index 4c44f1f1..98a126ac 100644 --- a/lib/docs/scrapers/mdn/css.rb +++ b/lib/docs/scrapers/mdn/css.rb @@ -4,7 +4,7 @@ module Docs self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/CSS' self.root_path = '/Reference' - html_filters.push 'css/clean_html', 'css/entries', 'title' + html_filters.push 'css/clean_html', 'css/entries' options[:root_title] = 'CSS' diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb index 943aeb5b..57828df3 100644 --- a/lib/docs/scrapers/mdn/dom.rb +++ b/lib/docs/scrapers/mdn/dom.rb @@ -4,7 +4,7 @@ module Docs self.name = 'DOM' self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API' - html_filters.push 'dom/clean_html', 'dom/entries', 'title' + html_filters.push 'dom/clean_html', 'dom/entries' options[:root_title] = 'DOM' diff --git a/lib/docs/scrapers/mdn/javascript.rb b/lib/docs/scrapers/mdn/javascript.rb index 935df61c..8238d3b4 100644 --- a/lib/docs/scrapers/mdn/javascript.rb +++ b/lib/docs/scrapers/mdn/javascript.rb @@ -6,7 +6,7 @@ module Docs self.name = 'JavaScript' self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference' - html_filters.push 'javascript/clean_html', 'javascript/entries', 'title' + html_filters.push 'javascript/clean_html', 'javascript/entries' options[:root_title] = 'JavaScript' diff --git a/lib/docs/scrapers/mdn/svg.rb b/lib/docs/scrapers/mdn/svg.rb index db9de7a1..4a9de544 100644 --- a/lib/docs/scrapers/mdn/svg.rb +++ b/lib/docs/scrapers/mdn/svg.rb @@ -6,7 +6,7 @@ module Docs self.name = 'SVG' self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/SVG' - html_filters.push 'svg/clean_html', 'svg/entries', 'title' + html_filters.push 'svg/clean_html', 'svg/entries' options[:root_title] = 'SVG' diff --git a/lib/docs/scrapers/mdn/xslt_xpath.rb b/lib/docs/scrapers/mdn/xslt_xpath.rb index 2407f0c0..286c48f4 100644 --- a/lib/docs/scrapers/mdn/xslt_xpath.rb +++ b/lib/docs/scrapers/mdn/xslt_xpath.rb @@ -6,15 +6,11 @@ module Docs self.root_path = '/XSLT' self.initial_paths = %w(/XPath) - html_filters.push 'xslt_xpath/clean_html', 'xslt_xpath/entries', 'title' + html_filters.push 'xslt_xpath/clean_html', 'xslt_xpath/entries' options[:root_title] = 'XSLT' options[:only_patterns] = [/\A\/XSLT/, /\A\/XPath/] - options[:fix_urls] = ->(url) do - url.sub! 'https://developer.mozilla.org/en-US/docs/Web/XSLT/Element', "#{XsltXpath.base_url}/XSLT" - url - end end end From e549e57431b80c1c969bf889d1cb753f78953eca Mon Sep 17 00:00:00 2001 From: Enoc Date: Thu, 6 May 2021 14:50:21 -0600 Subject: [PATCH 5/5] Remove comments with version(date) --- lib/docs/scrapers/mdn/dom.rb | 2 -- lib/docs/scrapers/mdn/html.rb | 2 -- 2 files changed, 4 deletions(-) diff --git a/lib/docs/scrapers/mdn/dom.rb b/lib/docs/scrapers/mdn/dom.rb index 57828df3..b75af630 100644 --- a/lib/docs/scrapers/mdn/dom.rb +++ b/lib/docs/scrapers/mdn/dom.rb @@ -8,7 +8,5 @@ module Docs options[:root_title] = 'DOM' - # self.release = '2021-04-29 21:55' - end end diff --git a/lib/docs/scrapers/mdn/html.rb b/lib/docs/scrapers/mdn/html.rb index d062bc88..1f108711 100644 --- a/lib/docs/scrapers/mdn/html.rb +++ b/lib/docs/scrapers/mdn/html.rb @@ -29,7 +29,5 @@ module Docs url end - # self.release = '2021-04-29 23:00' - end end