From adba5c1cac7db5a52afd9e0a2459edf070cfa756 Mon Sep 17 00:00:00 2001 From: Thibaut Courouble Date: Sun, 3 Apr 2016 11:19:10 -0400 Subject: [PATCH] Update URL stubs --- lib/docs/core/scraper.rb | 24 ------------------------ lib/docs/scrapers/angular.rb | 6 +----- lib/docs/scrapers/dojo.rb | 8 +++----- 3 files changed, 4 insertions(+), 34 deletions(-) diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb index 1d78a232..2d3b43ee 100644 --- a/lib/docs/core/scraper.rb +++ b/lib/docs/core/scraper.rb @@ -188,30 +188,6 @@ module Docs @pipeline = nil end - module StubRootPage - private - - def request_one(url) - stub_root_page if url == root_url.to_s - super - end - - def request_all(urls, &block) - stub_root_page - super - end - - def stub_root_page - response = Typhoeus::Response.new( - effective_url: root_url.to_s, - code: 200, - headers: { 'Content-Type' => 'text/html' }, - body: root_page_body) - - Typhoeus.stub(root_url.to_s).and_return(response) - end - end - module FixInternalUrlsBehavior def self.included(base) base.extend ClassMethods diff --git a/lib/docs/scrapers/angular.rb b/lib/docs/scrapers/angular.rb index 250b4779..c73e1b70 100644 --- a/lib/docs/scrapers/angular.rb +++ b/lib/docs/scrapers/angular.rb @@ -1,7 +1,5 @@ module Docs class Angular < UrlScraper - include StubRootPage - self.name = 'Angular.js' self.slug = 'angular' self.type = 'angular' @@ -59,9 +57,7 @@ module Docs self.base_url = "https://code.angularjs.org/#{release}/docs/partials/" end - private - - def root_page_body + stub '' do require 'capybara/dsl' Capybara.current_driver = :selenium Capybara.run_server = false diff --git a/lib/docs/scrapers/dojo.rb b/lib/docs/scrapers/dojo.rb index e1542091..c13bd704 100644 --- a/lib/docs/scrapers/dojo.rb +++ b/lib/docs/scrapers/dojo.rb @@ -2,8 +2,6 @@ require 'yajl/json_gem' module Docs class Dojo < UrlScraper - include StubRootPage - self.type = 'dojo' self.release = '1.10' self.base_url = "http://dojotoolkit.org/api/#{release}/" @@ -31,15 +29,15 @@ module Docs Licensed under the AFL 2.1 and BSD 3-Clause licenses. HTML - private - - def root_page_body + stub '' do response = request_one("#{self.base_url}tree.json") json = JSON.parse(response.body) urls = get_url_list(json) urls.map { |url| "#{url}" }.join end + private + def get_url_list(json, set = Set.new) set.add("#{self.class.base_url}#{json['fullname']}.html?xhr=true") json['children'].each { |child| get_url_list(child, set) } if json['children']