From 417ccbd9b4712a9191ee65b9fea898ef132c1809 Mon Sep 17 00:00:00 2001 From: doertedev Date: Fri, 13 Nov 2015 18:10:06 +0100 Subject: [PATCH] Moving from chef to chef client --- lib/docs/filters/chef/clean_html.rb | 10 ---- lib/docs/filters/chef/entries.rb | 10 ---- lib/docs/filters/chefclient/clean_html.rb | 11 ++++ lib/docs/filters/chefclient/entries.rb | 63 +++++++++++++++++++++++ lib/docs/scrapers/chef.rb | 27 ---------- lib/docs/scrapers/chefclient.rb | 28 ++++++++++ public/docs/docs.json | 2 +- 7 files changed, 103 insertions(+), 48 deletions(-) delete mode 100644 lib/docs/filters/chef/clean_html.rb delete mode 100644 lib/docs/filters/chef/entries.rb create mode 100644 lib/docs/filters/chefclient/clean_html.rb create mode 100644 lib/docs/filters/chefclient/entries.rb delete mode 100644 lib/docs/scrapers/chef.rb create mode 100644 lib/docs/scrapers/chefclient.rb diff --git a/lib/docs/filters/chef/clean_html.rb b/lib/docs/filters/chef/clean_html.rb deleted file mode 100644 index 5d79623b..00000000 --- a/lib/docs/filters/chef/clean_html.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Docs - class Chef - class CleanHtmlFilter < Filter - def call - css('h1 a', 'h2 a', 'h3 a','div.footer').remove - doc - end - end - end -end diff --git a/lib/docs/filters/chef/entries.rb b/lib/docs/filters/chef/entries.rb deleted file mode 100644 index 0c52fcf9..00000000 --- a/lib/docs/filters/chef/entries.rb +++ /dev/null @@ -1,10 +0,0 @@ -module Docs - class Chef - class EntriesFilter < Docs::EntriesFilter - def get_name - at_css('div.body h1 a').remove - at_css('div.body h1').content - end - end - end -end diff --git a/lib/docs/filters/chefclient/clean_html.rb b/lib/docs/filters/chefclient/clean_html.rb new file mode 100644 index 00000000..58726cc7 --- /dev/null +++ b/lib/docs/filters/chefclient/clean_html.rb @@ -0,0 +1,11 @@ +module Docs + class Chefclient + class CleanHtmlFilter < Filter + def call + css('h1 a, h2 a, h3 a').remove + doc = at_css('div.body[role="main"]') + doc + end + end + end +end diff --git a/lib/docs/filters/chefclient/entries.rb b/lib/docs/filters/chefclient/entries.rb new file mode 100644 index 00000000..af1f62f2 --- /dev/null +++ b/lib/docs/filters/chefclient/entries.rb @@ -0,0 +1,63 @@ +module Docs + class Chefclient + class EntriesFilter < Docs::EntriesFilter + + ADDITIONAL_URLS = [ + { + slug: 'knife_common_options', + type: 'Miscellaneous/Knife', + name: 'Common options' + }, + { + slug: 'knife_using', + type: 'Miscellaneous/Knife', + name: 'Using knife' + }, + { + slug: 'config_rb_knife_optional_settings', + type: 'Miscellaneous/Knife', + name: 'Optional settings' + }, + { + slug: 'resource_common', + name: 'About resources' + } + ] + + def get_name + ADDITIONAL_URLS.each do |url| + next unless slug == url[:slug] + return url[:name] + end + + css('.main-item a').map do |node| + next unless node.attributes['href'].to_s == slug + + return node.attributes['title'].to_s + end + #css('h1').try(:content) + return 'adsf' + end + + def get_type + ADDITIONAL_URLS.each do |url| + next unless slug == url[:slug] + next unless url[:type].nil? + return url[:type] + end + + css('.main-item a').map do |node| + next unless node.attributes['href'].to_s == slug + if node.ancestors('.sub-items')[0].attributes['class'].to_s.include? 'level-2' + level2 = node.ancestors('.sub-items')[0].previous_element['title'].to_s + level1 = node.ancestors('.sub-items')[0].ancestors('.sub-items')[0].previous_element['title'].to_s + return "#{level1}/#{level2}" + end + + return node.ancestors('.sub-items')[0].previous_element['title'].to_s + end + 'Miscellaneous' + end + end + end +end diff --git a/lib/docs/scrapers/chef.rb b/lib/docs/scrapers/chef.rb deleted file mode 100644 index 9ed4ae3c..00000000 --- a/lib/docs/scrapers/chef.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Docs - class Chef < UrlScraper - self.name = 'Chef' - self.slug = 'chef' - self.type = 'chef' - self.version = '12.5' - self.base_url = 'https://docs.chef.io/' - self.links = { - home: 'https://www.chef.io/', - docs: 'https://docs.chef.io/' - } - - html_filters.push 'chef/entries', 'chef/clean_html' - - options[:container] = '.bodywrapper' - - options[:only_patterns] = [/resource_.*.html/] - options[:skip_patterns] = [/resource_common\.html/] - - options[:trailing_slash] = false - - options[:attribution] = <<-HTML - © 2015 Chef Software, Inc.
- Creative Commons Attribution 3.0 Unported License. - HTML - end -end diff --git a/lib/docs/scrapers/chefclient.rb b/lib/docs/scrapers/chefclient.rb new file mode 100644 index 00000000..8eff8b68 --- /dev/null +++ b/lib/docs/scrapers/chefclient.rb @@ -0,0 +1,28 @@ +module Docs + class Chefclient < UrlScraper + self.name = 'Chef Client' + self.slug = 'chefclient' + self.type = 'chefclient' + self.version = '12.5' + self.base_url = "https://docs.chef.io/release/#{version.sub '.', '-'}/" + self.links = { + home: 'https://www.chef.io/', + docs: 'https://docs.chef.io/' + } + + html_filters.push 'chefclient/entries', 'chefclient/clean_html' + + options[:fix_urls] = ->(url) do + url.remove! %r{/release/[0-9\-+]/} + url + end + + options[:skip_patterns] = [/_images\//] + options[:trailing_slash] = false + + options[:attribution] = <<-HTML + © 2015 Chef Software, Inc.
+ Creative Commons Attribution 3.0 Unported License. + HTML + end +end diff --git a/public/docs/docs.json b/public/docs/docs.json index 0637a088..d79b2fc1 100644 --- a/public/docs/docs.json +++ b/public/docs/docs.json @@ -1 +1 @@ -[] \ No newline at end of file +[{"name":"Angular.js","slug":"angular","type":"angular","version":"1.4.7","index_path":"angular/index.json","db_path":"angular/db.json","links":null,"mtime":1444657214,"db_size":971848},{"name":"Apache HTTP Server","slug":"apache_http_server","type":"apache","version":"2.4.17","index_path":"apache_http_server/index.json","db_path":"apache_http_server/db.json","links":{"home":"http://httpd.apache.org/"},"mtime":1430087732,"db_size":2806035},{"name":"Backbone.js","slug":"backbone","type":"underscore","version":"1.2.3","index_path":"backbone/index.json","db_path":"backbone/db.json","links":{"home":"http://backbonejs.org/","code":"https://github.com/jashkenas/backbone"},"mtime":1442073277,"db_size":82033},{"name":"Bower","slug":"bower","type":"bower","version":"1.6.3","index_path":"bower/index.json","db_path":"bower/db.json","links":{"home":"http://bower.io/","code":"https://github.com/bower/bower"},"mtime":1445179203,"db_size":34591},{"name":"C","slug":"c","type":"c","version":null,"index_path":"c/index.json","db_path":"c/db.json","links":null,"mtime":1442071720,"db_size":3704145},{"name":"Chai","slug":"chai","type":"chai","version":"2.3.0","index_path":"chai/index.json","db_path":"chai/db.json","links":{"home":"http://chaijs.com/","code":"https://github.com/chaijs/chai"},"mtime":1430706732,"db_size":128007},{"name":"Chef Client","slug":"chefclient","type":"chefclient","version":"12.5","index_path":"chefclient/index.json","db_path":"chefclient/db.json","links":{"home":"https://www.chef.io/","docs":"https://docs.chef.io/"},"mtime":1447434534,"db_size":3937485},{"name":"Clojure","slug":"clojure","type":"clojure","version":"1.7","index_path":"clojure/index.json","db_path":"clojure/db.json","links":null,"mtime":1436101905,"db_size":524295},{"name":"CoffeeScript","slug":"coffeescript","type":"coffeescript","version":"1.10.0","index_path":"coffeescript/index.json","db_path":"coffeescript/db.json","links":{"home":"http://coffeescript.org","code":"https://github.com/jashkenas/coffeescript"},"mtime":1442073359,"db_size":53981},{"name":"Cordova","slug":"cordova","type":"cordova","version":"5.1.1","index_path":"cordova/index.json","db_path":"cordova/db.json","links":{"home":"http://cordova.apache.org/"},"mtime":1439135934,"db_size":580770},{"name":"C++","slug":"cpp","type":"c","version":null,"index_path":"cpp/index.json","db_path":"cpp/db.json","links":null,"mtime":1442072867,"db_size":15982350},{"name":"CSS","slug":"css","type":"mdn","version":null,"index_path":"css/index.json","db_path":"css/db.json","links":null,"mtime":1445184687,"db_size":5542116},{"name":"D3.js","slug":"d3","type":"d3","version":"3.5.6","index_path":"d3/index.json","db_path":"d3/db.json","links":{"home":"http://d3js.org/","code":"https://github.com/mbostock/d3"},"mtime":1436102054,"db_size":621700},{"name":"Django","slug":"django","type":"sphinx","version":"1.8.6","index_path":"django/index.json","db_path":"django/db.json","links":{"home":"https://www.djangoproject.com/","code":"https://github.com/django/django"},"mtime":1440862930,"db_size":4900789},{"name":"DOM","slug":"dom","type":"mdn","version":null,"index_path":"dom/index.json","db_path":"dom/db.json","links":null,"mtime":1445188189,"db_size":16899023},{"name":"DOM Events","slug":"dom_events","type":"mdn","version":null,"index_path":"dom_events/index.json","db_path":"dom_events/db.json","links":null,"mtime":1445185458,"db_size":1196362},{"name":"Drupal","slug":"drupal","type":"drupal","version":"7.40","index_path":"drupal/index.json","db_path":"drupal/db.json","links":{"home":"https://www.drupal.org/"},"mtime":1445179667,"db_size":15804364},{"name":"Ember.js","slug":"ember","type":"ember","version":"2.1.0","index_path":"ember/index.json","db_path":"ember/db.json","links":{"home":"http://emberjs.com/","code":"https://github.com/emberjs/ember.js"},"mtime":1444659821,"db_size":1194818},{"name":"Express","slug":"express","type":"express","version":"4.13.0","index_path":"express/index.json","db_path":"express/db.json","links":{"home":"http://expressjs.com/","code":"https://github.com/strongloop/express/"},"mtime":1436103414,"db_size":181622},{"name":"Git","slug":"git","type":"git","version":"2.6.1","index_path":"git/index.json","db_path":"git/db.json","links":{"home":"http://git-scm.com/","code":"https://github.com/git/git"},"mtime":1444659959,"db_size":1424351},{"name":"Go","slug":"go","type":"go","version":"1.5.0","index_path":"go/index.json","db_path":"go/db.json","links":{"home":"https://golang.org/","code":"https://go.googlesource.com/go"},"mtime":1440863110,"db_size":2724136},{"name":"Grunt","slug":"grunt","type":"grunt","version":"0.4.5","index_path":"grunt/index.json","db_path":"grunt/db.json","links":null,"mtime":1420139790,"db_size":116977},{"name":"Haskell","slug":"haskell","type":"haskell","version":"7.10.2","index_path":"haskell/index.json","db_path":"haskell/db.json","links":null,"mtime":1438638460,"db_size":6774322},{"name":"HTML","slug":"html","type":"mdn","version":null,"index_path":"html/index.json","db_path":"html/db.json","links":null,"mtime":1445184800,"db_size":2009435},{"name":"HTTP","slug":"http","type":"rfc","version":null,"index_path":"http/index.json","db_path":"http/db.json","links":null,"mtime":1424635284,"db_size":1183973},{"name":"io.js","slug":"iojs","type":"node","version":"3.3.1","index_path":"iojs/index.json","db_path":"iojs/db.json","links":{"home":"https://iojs.org/en/index.html","code":"https://github.com/iojs/io.js"},"mtime":1444660019,"db_size":651513},{"name":"JavaScript","slug":"javascript","type":"mdn","version":null,"index_path":"javascript/index.json","db_path":"javascript/db.json","links":null,"mtime":1445184936,"db_size":5168869},{"name":"jQuery","slug":"jquery","type":"jquery","version":"up to 2.1.0","index_path":"jquery/index.json","db_path":"jquery/db.json","links":null,"mtime":1420139791,"db_size":1783170},{"name":"jQuery Mobile","slug":"jquerymobile","type":"jquery","version":"1.4.0","index_path":"jquerymobile/index.json","db_path":"jquerymobile/db.json","links":null,"mtime":1420139791,"db_size":1033621},{"name":"jQuery UI","slug":"jqueryui","type":"jquery","version":"1.10.4","index_path":"jqueryui/index.json","db_path":"jqueryui/db.json","links":null,"mtime":1420139791,"db_size":826305},{"name":"Knockout.js","slug":"knockout","type":"knockout","version":"3.3.0","index_path":"knockout/index.json","db_path":"knockout/db.json","links":null,"mtime":1424535229,"db_size":394621},{"name":"Laravel","slug":"laravel","type":"laravel","version":"5.1.1","index_path":"laravel/index.json","db_path":"laravel/db.json","links":{"home":"http://laravel.com/","code":"https://github.com/laravel/laravel"},"mtime":1434309758,"db_size":6670433},{"name":"Less","slug":"less","type":"less","version":"2.5.0","index_path":"less/index.json","db_path":"less/db.json","links":{"home":"http://lesscss.org/","code":"https://github.com/less/less.js"},"mtime":1428071398,"db_size":144435},{"name":"lodash","slug":"lodash","type":"lodash","version":"3.10.0","index_path":"lodash/index.json","db_path":"lodash/db.json","links":{"home":"https://lodash.com/","code":"https://github.com/lodash/lodash/"},"mtime":1436103513,"db_size":245003},{"name":"Lua","slug":"lua","type":"lua","version":"5.3","index_path":"lua/index.json","db_path":"lua/db.json","links":null,"mtime":1424016672,"db_size":314005},{"name":"Marionette.js","slug":"marionette","type":"marionette","version":"2.4.3","index_path":"marionette/index.json","db_path":"marionette/db.json","links":{"home":"http://marionettejs.com/","code":"https://github.com/marionettejs/backbone.marionette"},"mtime":1442073591,"db_size":309831},{"name":"Markdown","slug":"markdown","type":"markdown","version":null,"index_path":"markdown/index.json","db_path":"markdown/db.json","links":null,"mtime":1420139791,"db_size":31338},{"name":"Meteor","slug":"meteor","type":"meteor","version":"1.2.0","index_path":"meteor/index.json","db_path":"meteor/db.json","links":{"home":"https://www.meteor.com/","code":"https://github.com/meteor/meteor/"},"mtime":1444660128,"db_size":427240},{"name":"Ruby / Minitest","slug":"minitest","type":"rdoc","version":"5.8.1","index_path":"minitest/index.json","db_path":"minitest/db.json","links":{"code":"https://github.com/seattlerb/minitest"},"mtime":1444662361,"db_size":166225},{"name":"mocha","slug":"mocha","type":"mocha","version":"2.3.3","index_path":"mocha/index.json","db_path":"mocha/db.json","links":{"home":"http://mochajs.org/","code":"https://github.com/mochajs/mocha"},"mtime":1444660211,"db_size":39062},{"name":"Modernizr","slug":"modernizr","type":"modernizr","version":"3.2.0","index_path":"modernizr/index.json","db_path":"modernizr/db.json","links":null,"mtime":1444661263,"db_size":72934},{"name":"Moment.js","slug":"moment","type":"moment","version":"2.10.2","index_path":"moment/index.json","db_path":"moment/db.json","links":null,"mtime":1429454707,"db_size":153981},{"name":"Mongoose","slug":"mongoose","type":"mongoose","version":"4.2.4","index_path":"mongoose/index.json","db_path":"mongoose/db.json","links":{"home":"http://mongoosejs.com/","code":"https://github.com/Automattic/mongoose"},"mtime":1442074298,"db_size":476844},{"name":"nginx","slug":"nginx","type":"nginx","version":"1.9.5","index_path":"nginx/index.json","db_path":"nginx/db.json","links":{"home":"http://nginx.org/","code":"http://hg.nginx.org/nginx"},"mtime":1444661375,"db_size":915788},{"name":"Node.js","slug":"node","type":"node","version":"5.0.0","index_path":"node/index.json","db_path":"node/db.json","links":{"home":"https://nodejs.org/","code":"https://github.com/nodejs/node"},"mtime":1445179081,"db_size":659476},{"name":"Nokogiri","slug":"nokogiri","type":"rdoc","version":"1.6.4","index_path":"nokogiri/index.json","db_path":"nokogiri/db.json","links":null,"mtime":1420139792,"db_size":463492},{"name":"npm","slug":"npm","type":"npm","version":"3.3.6","index_path":"npm/index.json","db_path":"npm/db.json","links":{"home":"https://www.npmjs.com/","code":"https://github.com/npm/npm"},"mtime":1444661467,"db_size":313300},{"name":"OpenTSDB","slug":"opentsdb","type":"opentsdb","version":"2.1.0","index_path":"opentsdb/index.json","db_path":"opentsdb/db.json","links":{"home":"http://opentsdb.net/","code":"https://github.com/OpenTSDB/opentsdb"},"mtime":1438531760,"db_size":624734},{"name":"Phalcon","slug":"phalcon","type":"phalcon","version":"2.0.6","index_path":"phalcon/index.json","db_path":"phalcon/db.json","links":{"home":"https://phalconphp.com/","code":"https://github.com/phalcon/cphalcon/"},"mtime":1442159941,"db_size":2650212},{"name":"Phaser","slug":"phaser","type":"phaser","version":"2.4.1","index_path":"phaser/index.json","db_path":"phaser/db.json","links":{"home":"http://phaser.io/","code":"https://github.com/photonstorm/phaser"},"mtime":1437839844,"db_size":6560962},{"name":"PHP","slug":"php","type":"php","version":"up to 5.6.13","index_path":"php/index.json","db_path":"php/db.json","links":null,"mtime":1442074610,"db_size":21518994},{"name":"PHPUnit","slug":"phpunit","type":"phpunit","version":"5.0","index_path":"phpunit/index.json","db_path":"phpunit/db.json","links":{"home":"https://phpunit.de/","code":"https://github.com/sebastianbergmann/phpunit"},"mtime":1444661714,"db_size":388192},{"name":"PostgreSQL","slug":"postgresql","type":"postgres","version":"9.4","index_path":"postgresql/index.json","db_path":"postgresql/db.json","links":null,"mtime":1420139795,"db_size":3815543},{"name":"Python","slug":"python","type":"sphinx","version":"3.5.0","index_path":"python/index.json","db_path":"python/db.json","links":null,"mtime":1444661855,"db_size":7460408},{"name":"Python 2","slug":"python2","type":"sphinx","version":"2.7.10","index_path":"python2/index.json","db_path":"python2/db.json","links":null,"mtime":1442070616,"db_size":6261019},{"name":"Q","slug":"q","type":"q","version":"1.4.1","index_path":"q/index.json","db_path":"q/db.json","links":{"home":"http://documentup.com/kriskowal/q/","code":"https://github.com/kriskowal/q"},"mtime":1438534016,"db_size":30006},{"name":"Ruby on Rails","slug":"rails","type":"rdoc","version":"4.2.3","index_path":"rails/index.json","db_path":"rails/db.json","links":{"home":"http://rubyonrails.org/","code":"https://github.com/rails/rails"},"mtime":1436105620,"db_size":3694909},{"name":"React","slug":"react","type":"react","version":"0.14.2","index_path":"react/index.json","db_path":"react/db.json","links":{"home":"https://facebook.github.io/react/","code":"https://github.com/facebook/react"},"mtime":1444662028,"db_size":295079},{"name":"React Native","slug":"react_native","type":"react","version":"0.14.1","index_path":"react_native/index.json","db_path":"react_native/db.json","links":{"home":"https://facebook.github.io/react-native/","code":"https://github.com/facebook/react-native"},"mtime":1444662142,"db_size":536376},{"name":"Redis","slug":"redis","type":"redis","version":"up to 3.0.5","index_path":"redis/index.json","db_path":"redis/db.json","links":{"home":"http://redis.io/","code":"https://github.com/antirez/redis"},"mtime":1445179278,"db_size":594228},{"name":"RequireJS","slug":"requirejs","type":"requirejs","version":"2.1.18","index_path":"requirejs/index.json","db_path":"requirejs/db.json","links":{"home":"http://requirejs.org/","code":"https://github.com/jrburke/requirejs"},"mtime":1433708935,"db_size":169460},{"name":"RethinkDB","slug":"rethinkdb","type":"rethinkdb","version":"2.1.2","index_path":"rethinkdb/index.json","db_path":"rethinkdb/db.json","links":{"home":"http://rethinkdb.com/","code":"https://github.com/rethinkdb/rethinkdb"},"mtime":1440865256,"db_size":502582},{"name":"Ruby","slug":"ruby","type":"rdoc","version":"2.2.1","index_path":"ruby/index.json","db_path":"ruby/db.json","links":{"home":"https://www.ruby-lang.org/","code":"https://github.com/ruby/ruby"},"mtime":1427644483,"db_size":6652108},{"name":"Rust","slug":"rust","type":"rust","version":"1.4.0","index_path":"rust/index.json","db_path":"rust/db.json","links":{"home":"http://www.rust-lang.org/","code":"https://github.com/rust-lang/rust"},"mtime":1444662470,"db_size":14125397},{"name":"Sass","slug":"sass","type":"yard","version":"3.4.17","index_path":"sass/index.json","db_path":"sass/db.json","links":null,"mtime":1442074420,"db_size":207303},{"name":"Sinon","slug":"sinon","type":"sinon","version":"1.17.1","index_path":"sinon/index.json","db_path":"sinon/db.json","links":{"home":"http://sinonjs.org/","code":"https://github.com/cjohansen/Sinon.JS"},"mtime":1444662527,"db_size":74956},{"name":"Socket.IO","slug":"socketio","type":"socketio","version":"1.3.6","index_path":"socketio/index.json","db_path":"socketio/db.json","links":{"home":"http://socket.io/","code":"https://github.com/socketio/socket.io"},"mtime":1437839408,"db_size":42638},{"name":"SVG","slug":"svg","type":"mdn","version":null,"index_path":"svg/index.json","db_path":"svg/db.json","links":null,"mtime":1438524102,"db_size":1631690},{"name":"Symfony","slug":"symfony","type":"laravel","version":"2.7","index_path":"symfony/index.json","db_path":"symfony/db.json","links":{"home":"http://symfony.com/","code":"https://github.com/symfony/symfony"},"mtime":1436103152,"db_size":11762612},{"name":"Underscore.js","slug":"underscore","type":"underscore","version":"1.8.3","index_path":"underscore/index.json","db_path":"underscore/db.json","links":null,"mtime":1437930061,"db_size":52589},{"name":"Vue.js","slug":"vue","type":"vue","version":"1.0.7","index_path":"vue/index.json","db_path":"vue/db.json","links":{"home":"http://vuejs.org/","code":"https://github.com/yyx990803/vue"},"mtime":1442073643,"db_size":214935},{"name":"webpack","slug":"webpack","type":"webpack","version":"1.12","index_path":"webpack/index.json","db_path":"webpack/db.json","links":{"home":"https://webpack.github.io/","code":"https://github.com/webpack/webpack"},"mtime":1440865499,"db_size":305761},{"name":"XPath","slug":"xpath","type":"mdn","version":null,"index_path":"xpath/index.json","db_path":"xpath/db.json","links":null,"mtime":1438524154,"db_size":93631},{"name":"Yii","slug":"yii","type":"yii","version":"2.0.2","index_path":"yii/index.json","db_path":"yii/db.json","links":null,"mtime":1423414038,"db_size":14309051},{"name":"Yii 1","slug":"yii1","type":"yii","version":"1.1.16","index_path":"yii1/index.json","db_path":"yii1/db.json","links":null,"mtime":1424012029,"db_size":13066030}] \ No newline at end of file