From 64eb1c86fc455f1dda85bcb7ea69fac7ed628091 Mon Sep 17 00:00:00 2001
From: Thibaut Courouble <thibaut@me.com>
Date: Mon, 5 Sep 2016 15:21:20 -0400
Subject: [PATCH] Log errors instead of exiting when scraping docs

---
 lib/docs.rb                            | 3 +++
 lib/docs/core/scraper.rb               | 8 ++++++--
 lib/docs/subscribers/doc_subscriber.rb | 9 +++++++++
 lib/tasks/docs.thor                    | 3 +++
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/docs.rb b/lib/docs.rb
index 2f0d8a29..8035f4c8 100644
--- a/lib/docs.rb
+++ b/lib/docs.rb
@@ -25,6 +25,9 @@ module Docs
   mattr_accessor :store_path
   self.store_path = File.expand_path '../public/docs', @@root_path
 
+  mattr_accessor :rescue_errors
+  self.rescue_errors = false
+
   class DocNotFound < NameError; end
 
   def self.all
diff --git a/lib/docs/core/scraper.rb b/lib/docs/core/scraper.rb
index 93aed4c0..d5109332 100644
--- a/lib/docs/core/scraper.rb
+++ b/lib/docs/core/scraper.rb
@@ -163,8 +163,12 @@ module Docs
         instrument 'ignore_response.scraper', response: response
       end
     rescue => e
-      puts "URL: #{response.url}"
-      raise e
+      if Docs.rescue_errors
+        instrument 'error.doc', exception: e, url: response.url
+        nil
+      else
+        raise e
+      end
     end
 
     def process_response(response)
diff --git a/lib/docs/subscribers/doc_subscriber.rb b/lib/docs/subscribers/doc_subscriber.rb
index 7c1fa550..18ad591d 100644
--- a/lib/docs/subscribers/doc_subscriber.rb
+++ b/lib/docs/subscribers/doc_subscriber.rb
@@ -20,6 +20,15 @@ module Docs
       log event.payload[:msg]
     end
 
+    def error(event)
+      exception = event.payload[:exception]
+      log "ERROR:"
+      puts "  #{event.payload[:url]}"
+      puts "  #{exception.class}: #{exception.message.gsub("\n", "\n    ")}"
+      puts exception.backtrace.select { |line| line.start_with?(Docs.root_path) }.join("\n  ").prepend("\n  ")
+      puts "\n"
+    end
+
     private
 
     def parse_payload(event)
diff --git a/lib/tasks/docs.thor b/lib/tasks/docs.thor
index 5739482f..7bff5521 100644
--- a/lib/tasks/docs.thor
+++ b/lib/tasks/docs.thor
@@ -58,6 +58,7 @@ class DocsCLI < Thor
   option :force, type: :boolean
   option :package, type: :boolean
   def generate(name)
+    Docs.rescue_errors = true
     Docs.install_report :store if options[:verbose]
     Docs.install_report :scraper if options[:debug]
     Docs.install_report :progress_bar, :doc if $stdout.tty?
@@ -93,6 +94,8 @@ class DocsCLI < Thor
     generate_manifest if result
   rescue Docs::DocNotFound => error
     handle_doc_not_found_error(error)
+  ensure
+    Docs.rescue_errors = false
   end
 
   desc 'manifest', 'Create the manifest'