Raise error and stop scraping on 4xx/5xx status code

lib/docs/core/response.rb

@@ -4,6 +4,10 @@ module Docs
       code == 200
     end
 
+    def error?
+      code != 404 && code >= 400 && code <= 599
+    end
+
     def empty?
       body.empty?
     end

lib/docs/core/scrapers/url_scraper.rb

@@ -29,6 +29,10 @@ module Docs
     end
 
     def process_response?(response)
+      if response.error?
+        raise "Error status code (#{response.code}): #{response.url}"
+      end
+
       response.success? && response.html? && base_url.contains?(response.effective_url)
     end
 

test/lib/docs/core/response_test.rb

@@ -29,6 +29,28 @@ class DocsResponseTest < MiniTest::Spec
     end
   end
 
+  describe "#error?" do
+    it "returns false when the code is 200" do
+      options.code = 200
+      refute response.error?
+    end
+
+    it "returns false when the code is 404" do
+      options.code = 404
+      refute response.error?
+    end
+
+    it "returns true when the code is 400" do
+      options.code = 400
+      assert response.error?
+    end
+
+    it "returns true when the code is 500" do
+      options.code = 500
+      assert response.error?
+    end
+  end
+
   describe "#empty?" do
     it "returns true when the body is empty" do
       options.body = ''

test/lib/docs/core/scrapers/url_scraper_test.rb

@@ -89,13 +89,18 @@ class DocsUrlScraperTest < MiniTest::Spec
 
   describe "#process_response?" do
     let :response do
-      OpenStruct.new success?: true, html?: true, effective_url: scraper.root_url
+      OpenStruct.new success?: true, html?: true, effective_url: scraper.root_url, error?: false
     end
 
     let :result do
       scraper.send :process_response?, response
     end
 
+    it "raises when the response is an error" do
+      response.send 'error?=', true
+      assert_raises(RuntimeError) { result }
+    end
+
     it "returns false when the response isn't successful" do
       response.send 'success?=', false
       refute result