Log errors instead of exiting when scraping docs

lib/docs.rb

@@ -25,6 +25,9 @@ module Docs
   mattr_accessor :store_path
   self.store_path = File.expand_path '../public/docs', @@root_path
 
+  mattr_accessor :rescue_errors
+  self.rescue_errors = false
+
   class DocNotFound < NameError; end
 
   def self.all

lib/docs/core/scraper.rb

@@ -163,9 +163,13 @@ module Docs
         instrument 'ignore_response.scraper', response: response
       end
     rescue => e
-      puts "URL: #{response.url}"
+      if Docs.rescue_errors
+        instrument 'error.doc', exception: e, url: response.url
+        nil
+      else
         raise e
       end
+    end
 
     def process_response(response)
       data = {}

lib/docs/subscribers/doc_subscriber.rb

@@ -20,6 +20,15 @@ module Docs
       log event.payload[:msg]
     end
 
+    def error(event)
+      exception = event.payload[:exception]
+      log "ERROR:"
+      puts "  #{event.payload[:url]}"
+      puts "  #{exception.class}: #{exception.message.gsub("\n", "\n    ")}"
+      puts exception.backtrace.select { |line| line.start_with?(Docs.root_path) }.join("\n  ").prepend("\n  ")
+      puts "\n"
+    end
+
     private
 
     def parse_payload(event)

lib/tasks/docs.thor

@@ -58,6 +58,7 @@ class DocsCLI < Thor
   option :force, type: :boolean
   option :package, type: :boolean
   def generate(name)
+    Docs.rescue_errors = true
     Docs.install_report :store if options[:verbose]
     Docs.install_report :scraper if options[:debug]
     Docs.install_report :progress_bar, :doc if $stdout.tty?
@@ -93,6 +94,8 @@ class DocsCLI < Thor
     generate_manifest if result
   rescue Docs::DocNotFound => error
     handle_doc_not_found_error(error)
+  ensure
+    Docs.rescue_errors = false
   end
 
   desc 'manifest', 'Create the manifest'