From f8b7cb738ee00472076fffc698b84488dd0ef92d Mon Sep 17 00:00:00 2001 From: Aditya Ardiya Date: Sat, 18 Dec 2021 05:24:19 +0900 Subject: [PATCH 1/7] Initial work for adding PointCloudLibrary --- .../filters/point_cloud_library/entries.rb | 43 +++++++++++++++++++ lib/docs/scrapers/point_cloud_library.rb | 32 ++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 lib/docs/filters/point_cloud_library/entries.rb create mode 100644 lib/docs/scrapers/point_cloud_library.rb diff --git a/lib/docs/filters/point_cloud_library/entries.rb b/lib/docs/filters/point_cloud_library/entries.rb new file mode 100644 index 00000000..2942b01f --- /dev/null +++ b/lib/docs/filters/point_cloud_library/entries.rb @@ -0,0 +1,43 @@ +module Docs + class PointCloudLibrary + class EntriesFilter < Docs::EntriesFilter + def get_type + if slug.start_with?("group") + 'Group' + else + 'Others' + end + end + + def additional_entries + return [] if root_page? + + css('table.memberdecls td.memItemRight').map do |node| + # Get the type of the entry from Doxygen table heading + type = node.parent.parent.css("tr.heading").text.strip + if type == 'Additional Inherited Members' then + return [] + end + + # Retrieve HREF link + first_link = node.css("a").first + if first_link.nil? then + return [] + end + href = first_link['href'] + if href.index("#").nil? then + # If it doesn't have #, it means it's linking to other page. + # So append # at the end to make it work + href += "#" + end + + [node.content, href, type] + end + end + + def include_default_entry? + !at_css('.obsolete') + end + end + end +end diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb new file mode 100644 index 00000000..b289795e --- /dev/null +++ b/lib/docs/scrapers/point_cloud_library.rb @@ -0,0 +1,32 @@ +module Docs + class PointCloudLibrary < UrlScraper + self.name = 'PointCloudLibrary' + self.type = 'point_cloud_library' + self.slug = 'point_cloud_library' + self.base_url = 'https://pointclouds.org/documentation/' + self.root_path = 'modules.html' + + self.links = { + home: 'https://pointclouds.org/', + code: 'https://github.com/PointCloudLibrary/pcl' + } + + html_filters.push 'point_cloud_library/entries' + + # Remove the `clean_text` because Doxygen are actually creating empty + # anchor such as to do anchor link.. and that anchor + # will be removed by clean_text + self.text_filters = FilterStack.new + text_filters.push 'images', 'inner_html' + + def get_latest_version(opts) + get_latest_github_release('PointCloudLibrary', 'pcl', opts) + end + + options[:container] = '.contents' + + # Skip source code since it doesn't provide any useful docs + options[:skip_patterns] = [/_source/] + + end +end From 46d2fd2ba0d1a8e01affb7824d0f8742eae20ad7 Mon Sep 17 00:00:00 2001 From: Aditya Ardiya Date: Wed, 29 Dec 2021 02:28:57 +0900 Subject: [PATCH 2/7] Simplify entries to just from PointCloudLibrary modules (group__*.html) --- .../filters/point_cloud_library/entries.rb | 54 +++++++++++-------- lib/docs/scrapers/point_cloud_library.rb | 6 ++- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/docs/filters/point_cloud_library/entries.rb b/lib/docs/filters/point_cloud_library/entries.rb index 2942b01f..2818d92c 100644 --- a/lib/docs/filters/point_cloud_library/entries.rb +++ b/lib/docs/filters/point_cloud_library/entries.rb @@ -2,41 +2,51 @@ module Docs class PointCloudLibrary class EntriesFilter < Docs::EntriesFilter def get_type - if slug.start_with?("group") - 'Group' + if slug.include?("group__") then + tmp = slug.dup + tmp.sub! "group__", "" + tmp.sub! "__", " " + tmp else - 'Others' + "‎" + end + end + + def get_name + type = get_type() + if type == "‎" + slug + else + "Module " + type end end def additional_entries - return [] if root_page? + # Only add additional_entries from PointCloudLibrary modules (group__*.html) + return [] if not slug.include?("group") + entries = [] css('table.memberdecls td.memItemRight').map do |node| - # Get the type of the entry from Doxygen table heading - type = node.parent.parent.css("tr.heading").text.strip - if type == 'Additional Inherited Members' then - return [] + href = node.at_css("a").attr('href') + if href.index("#").nil? then + href += "#" end - # Retrieve HREF link - first_link = node.css("a").first - if first_link.nil? then - return [] + # Skip page that's not crawled + # TODO: Sync this with options[:skip_patterns] in point_cloud_library.rb + if href.include?("namespace") || href.include?("structsvm") || href.include?("classopenni") then + next end - href = first_link['href'] - if href.index("#").nil? then - # If it doesn't have #, it means it's linking to other page. - # So append # at the end to make it work - href += "#" + + # Only add function and classes documentation + doxygen_type = node.parent.parent.at_css("tr.heading").text.strip + if not(doxygen_type == "Functions" || doxygen_type == "Classes") then + next end - [node.content, href, type] + entries << [node.content, href] end - end - - def include_default_entry? - !at_css('.obsolete') + entries end end end diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb index b289795e..138c57a5 100644 --- a/lib/docs/scrapers/point_cloud_library.rb +++ b/lib/docs/scrapers/point_cloud_library.rb @@ -5,6 +5,10 @@ module Docs self.slug = 'point_cloud_library' self.base_url = 'https://pointclouds.org/documentation/' self.root_path = 'modules.html' + # Add hierarchy.html to crawl all classes*.html that's not reachable from modules.html + self.initial_paths = [ + "https://pointclouds.org/documentation/hierarchy.html" + ] self.links = { home: 'https://pointclouds.org/', @@ -26,7 +30,7 @@ module Docs options[:container] = '.contents' # Skip source code since it doesn't provide any useful docs - options[:skip_patterns] = [/_source/] + options[:skip_patterns] = [/_source/, /namespace/, /h\.html/, /structsvm/, /struct_/, /classopenni/, /class_/] end end From 5632820da94b120563395f72af40fc8895b1132a Mon Sep 17 00:00:00 2001 From: Aditya Ardiya Date: Wed, 29 Dec 2021 21:20:11 +0900 Subject: [PATCH 3/7] Add PointCloudLibrary icons --- public/icons/docs/point_cloud_library/16.png | Bin 0 -> 494 bytes public/icons/docs/point_cloud_library/16@2x.png | Bin 0 -> 894 bytes public/icons/docs/point_cloud_library/SOURCE | 1 + 3 files changed, 1 insertion(+) create mode 100644 public/icons/docs/point_cloud_library/16.png create mode 100644 public/icons/docs/point_cloud_library/16@2x.png create mode 100644 public/icons/docs/point_cloud_library/SOURCE diff --git a/public/icons/docs/point_cloud_library/16.png b/public/icons/docs/point_cloud_library/16.png new file mode 100644 index 0000000000000000000000000000000000000000..90f0b827a7f2a191f88604a8dcac88be7a6c5f88 GIT binary patch literal 494 zcmVd>XqrGtqeK>}vz zpcs5U2kX$Gqws@L5JZtAi_xLbMVv%KA-^ILNbS<4i(9+ZF71#)K-0W;ZXyqb)?4rH z^W}TQ9V$A1iBL|Gj9~(y_FiUoE4O8NUHY)n&+f|=N@D_gDKwVTJ8L9ltu-U#=Wis9 zl2Az2`(o|g!t-|**O2Wazqg&x0j2d{pHs)lr{5mD&b)8kyk@O6{pn^R8E z5Fg^q(v!+x06^LKUeS3k?t1Mul|FiflJoNb5CZ@SDRi+B4Q8|1XU5160&G;P)vu8y z{!n@h0I_1tn*s5NVQ?0O7!#~YOdA6Nn7!$B0-knvDOYlSK4yry<%!BP2}L0i*8nt) z!B*6JHCg#-Pwf0i=Z8btd58bAMN}o{)s4XKWw)5S7og@X2>e33pN|g@48A2bD)8c; k+GMCuJ%BzvY&JKq03Gd#)cSW@4*&oF07*qoM6N<$f-D@<2mk;8 literal 0 HcmV?d00001 diff --git a/public/icons/docs/point_cloud_library/16@2x.png b/public/icons/docs/point_cloud_library/16@2x.png new file mode 100644 index 0000000000000000000000000000000000000000..f56f6b4b505c267b2e8c478ed4d449e6d7167bc4 GIT binary patch literal 894 zcmV-^1A+XBP)|TvZf?zyD5VLPLsRu?YmU8xa%CAT~r7neeFuIh?q>uG!v~DU9{b#mBiMOlv1V8QmCL67cK%;f;8Yp9q_jvQM_mg*fW zoX^A;X8j20WuRncB^)z&S|pwaJt6qGziVW>gxwZ=@XO4DGV3v;&BRqn_hGgL{I+az z^y{1Fhfbz@42>Zj0jBZ9&H0}Qqsd&)N59qu&rkKW`o-`7##Z1vj9-u?Bt2y2pS>0M zuLX9zXfO&~{Ik^#tQ9t_c;e0er-tX2g&#Qa>iWp}(`I%p<}B0KS^T#0?C{}jvEJSz z(Q@F5f*^1~5co>HuH*6xVEZLzZrFTsclRgPZwVhfnLY$+pSStVONVY(@0ZdoF|RY2 z9nE$Z2P%D6De_Z&&7hZnJ*x=@c&)QIBjYWkOb`TBX0`BPxgCgy-k(`ba5cX{=YTuc zZ=T*z=`~ae^MrDs#>m$SN~KawQnbfRj{;w8+_>r8%6nc;F60h#P!1E-IR}4yXh?ej zAB*CT)PhTXALtEX~*9zd?8Q;EkQPC;MwP1Q6*Ev$E z6$EtL6#aNDLpE6)Gw&Y8X!T+E)qCH(zQ#4Ji4A}n<>Fo0?t?$n0me+Hm!2hesvckN znZiXBZ2{U6b8|Uo#&eP%+|`-CQcnQD;$jql_L?zaMs9rk^F#G|v&j!H1H&eI8zk<< z__1Cyz~tm)83ER^O12h6k;aSu!Wes$^w{NuvYX7*|0g3Iqr9sCX8ca~B) Ua$YUyLI3~&07*qoM6N<$f@%exTmS$7 literal 0 HcmV?d00001 diff --git a/public/icons/docs/point_cloud_library/SOURCE b/public/icons/docs/point_cloud_library/SOURCE new file mode 100644 index 00000000..180ad4e2 --- /dev/null +++ b/public/icons/docs/point_cloud_library/SOURCE @@ -0,0 +1 @@ +https://github.com/PointCloudLibrary/pcl/blob/master/pcl.png From b1fef165c433b990ae7fac9c1b84590f4be67392 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Wed, 29 Dec 2021 20:07:30 +0100 Subject: [PATCH 4/7] point_cloud_library: attribution --- assets/javascripts/templates/pages/about_tmpl.coffee | 5 +++++ lib/docs/scrapers/point_cloud_library.rb | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/assets/javascripts/templates/pages/about_tmpl.coffee b/assets/javascripts/templates/pages/about_tmpl.coffee index 97a24878..f9925f53 100644 --- a/assets/javascripts/templates/pages/about_tmpl.coffee +++ b/assets/javascripts/templates/pages/about_tmpl.coffee @@ -657,6 +657,11 @@ credits = [ '2005-2017 Sebastian Bergmann', 'CC BY', 'https://creativecommons.org/licenses/by/3.0/' + ], [ + 'PointCloudLibrary', + '2009–2012, Willow Garage, Inc.
© 2012–, Open Perception, Inc.', + 'BSD', + 'https://raw.githubusercontent.com/PointCloudLibrary/pcl/master/LICENSE.txt' ], [ 'Pony', '2016-2020, The Pony Developers & 2014-2015, Causality Ltd.', diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb index 138c57a5..a30d1d65 100644 --- a/lib/docs/scrapers/point_cloud_library.rb +++ b/lib/docs/scrapers/point_cloud_library.rb @@ -21,13 +21,18 @@ module Docs # anchor such as to do anchor link.. and that anchor # will be removed by clean_text self.text_filters = FilterStack.new - text_filters.push 'images', 'inner_html' + text_filters.push 'images', 'inner_html', 'attribution' def get_latest_version(opts) get_latest_github_release('PointCloudLibrary', 'pcl', opts) end options[:container] = '.contents' + options[:attribution] = <<-HTML + © 2009–2012, Willow Garage, Inc.
+ © 2012–, Open Perception, Inc.
+ Licensed under the BSD License. + HTML # Skip source code since it doesn't provide any useful docs options[:skip_patterns] = [/_source/, /namespace/, /h\.html/, /structsvm/, /struct_/, /classopenni/, /class_/] From 300d2f823587aaff94084321bde7d373180b85a1 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Wed, 29 Dec 2021 20:10:09 +0100 Subject: [PATCH 5/7] point_cloud_library: clean_html, entries --- .../filters/point_cloud_library/clean_html.rb | 12 +++++++++++ .../filters/point_cloud_library/entries.rb | 20 ++++++------------- lib/docs/scrapers/point_cloud_library.rb | 3 +-- 3 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 lib/docs/filters/point_cloud_library/clean_html.rb diff --git a/lib/docs/filters/point_cloud_library/clean_html.rb b/lib/docs/filters/point_cloud_library/clean_html.rb new file mode 100644 index 00000000..143345d6 --- /dev/null +++ b/lib/docs/filters/point_cloud_library/clean_html.rb @@ -0,0 +1,12 @@ +module Docs + class PointCloudLibrary + class CleanHtmlFilter < Filter + def call + @doc = at_css('.contents') + css('.dynheader.closed').remove + css('.permalink').remove + doc + end + end + end +end diff --git a/lib/docs/filters/point_cloud_library/entries.rb b/lib/docs/filters/point_cloud_library/entries.rb index 2818d92c..29b22214 100644 --- a/lib/docs/filters/point_cloud_library/entries.rb +++ b/lib/docs/filters/point_cloud_library/entries.rb @@ -2,23 +2,15 @@ module Docs class PointCloudLibrary class EntriesFilter < Docs::EntriesFilter def get_type - if slug.include?("group__") then - tmp = slug.dup - tmp.sub! "group__", "" - tmp.sub! "__", " " - tmp - else - "‎" - end + group = at_css('.title .ingroups') + return group.content unless group.nil? + name = get_name + return 'pcl' unless name.match(/^pcl::.*::/) + name.gsub(/^pcl::/, '').gsub(/::.*/, '') end def get_name - type = get_type() - if type == "‎" - slug - else - "Module " + type - end + at_css('.title').content.gsub(/[<(].*/, '') end def additional_entries diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb index a30d1d65..c2b04431 100644 --- a/lib/docs/scrapers/point_cloud_library.rb +++ b/lib/docs/scrapers/point_cloud_library.rb @@ -15,7 +15,7 @@ module Docs code: 'https://github.com/PointCloudLibrary/pcl' } - html_filters.push 'point_cloud_library/entries' + html_filters.push 'point_cloud_library/entries', 'point_cloud_library/clean_html' # Remove the `clean_text` because Doxygen are actually creating empty # anchor such as to do anchor link.. and that anchor @@ -27,7 +27,6 @@ module Docs get_latest_github_release('PointCloudLibrary', 'pcl', opts) end - options[:container] = '.contents' options[:attribution] = <<-HTML © 2009–2012, Willow Garage, Inc.
© 2012–, Open Perception, Inc.
From f6e71bad459fcee8aac85462991d9df236c2dddb Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Wed, 29 Dec 2021 20:10:09 +0100 Subject: [PATCH 6/7] point_cloud_library: release --- lib/docs/scrapers/point_cloud_library.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/docs/scrapers/point_cloud_library.rb b/lib/docs/scrapers/point_cloud_library.rb index c2b04431..f9ce6827 100644 --- a/lib/docs/scrapers/point_cloud_library.rb +++ b/lib/docs/scrapers/point_cloud_library.rb @@ -9,6 +9,7 @@ module Docs self.initial_paths = [ "https://pointclouds.org/documentation/hierarchy.html" ] + self.release = '1.12.1' self.links = { home: 'https://pointclouds.org/', @@ -24,7 +25,7 @@ module Docs text_filters.push 'images', 'inner_html', 'attribution' def get_latest_version(opts) - get_latest_github_release('PointCloudLibrary', 'pcl', opts) + get_latest_github_release('PointCloudLibrary', 'pcl', opts)[4..] end options[:attribution] = <<-HTML From a0e3b6f95c632720e0b7e678a7ba78c61b187358 Mon Sep 17 00:00:00 2001 From: Simon Legner Date: Wed, 29 Dec 2021 20:14:10 +0100 Subject: [PATCH 7/7] point_cloud_library: news --- assets/javascripts/news.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assets/javascripts/news.json b/assets/javascripts/news.json index f047ba73..bfc6225c 100644 --- a/assets/javascripts/news.json +++ b/assets/javascripts/news.json @@ -1,4 +1,8 @@ [ + [ + "2021-12-29", + "New documentation: PointCloudLibrary" + ], [ "2021-12-27", "New documentation: Zig"