diff --git a/lib/docs/core/scrapers/url_scraper.rb b/lib/docs/core/scrapers/url_scraper.rb index 043365e9..1daa1c05 100644 --- a/lib/docs/core/scrapers/url_scraper.rb +++ b/lib/docs/core/scrapers/url_scraper.rb @@ -13,6 +13,8 @@ module Docs end end + @@rate_limiter = nil + self.params = {} self.headers = { 'User-Agent' => 'DevDocs' } self.force_gzip = false @@ -24,6 +26,15 @@ module Docs end def request_all(urls, &block) + if options[:rate_limit] + if @@rate_limiter + @@rate_limiter.limit = options[:rate_limit] + else + @@rate_limiter = RateLimiter.new(options[:rate_limit]) + Typhoeus.before(&@@rate_limiter.to_proc) + end + end + Requester.run urls, request_options: request_options, &block end @@ -165,5 +176,35 @@ module Docs super.merge! redirections: self.class.redirections end end + + class RateLimiter + attr_accessor :limit + + def initialize(limit) + @limit = limit + @minute = nil + @counter = 0 + end + + def call(*) + if @minute != Time.now.min + @minute = Time.now.min + @counter = 0 + end + + @counter += 1 + + if @counter >= @limit + wait = Time.now.end_of_minute.to_i - Time.now.to_i + 1 + sleep wait + end + + true + end + + def to_proc + method(:call).to_proc + end + end end end