add #sitemaps method

README.rdoc

@@ -8,6 +8,7 @@ Usage:
     robotex = Robotex.new "My User Agent"
     robotex.allowed?("http://www.example.com/foo")
     robotex.delay!("http://www.example.com/foo") # wait until any specified Crawl-Delay has passed
+    robotex.sitemaps("http://www.example.com/") # return an array of sitemap urls
 
 == Acknowledgements
 

lib/robotex.rb

@@ -25,6 +25,7 @@ def initialize(uri, user_agent)
       @disallows = {}
       @allows = {}
       @delays = {}
+      @sitemaps = []
       agent = /.*/
       io.each do |line|
         next if line =~ /^\s*(#.*|$)/
@@ -43,6 +44,8 @@ def initialize(uri, user_agent)
             @disallows[agent] << to_regex(value)
           when "crawl-delay"
             @delays[agent] = value.to_i
+          when "sitemap"
+            @sitemaps << value
         end
       end
 
@@ -86,6 +89,10 @@ def delay(user_agent)
       end
       nil
     end
+
+    def sitemaps(uri)
+      @sitemaps
+    end
 
     protected
 
@@ -148,5 +155,11 @@ def delay!(uri)
     sleep delay - (Time.now - @last_accessed) if !!delay
     @last_accessed = Time.now
   end
-
+
+  #
+  # Returns an array of the sitemap urls specified in robots.txt
+  #
+  def sitemaps(uri)
+    parse_host(uri).sitemaps(uri)
+  end
 end

spec/robotex_spec.rb

@@ -5,6 +5,9 @@
   before(:all) do
     FakeWeb.allow_net_connect = false
     robots = <<-END
+Sitemap: http://www.example.com/sitemap_1.xml
+Sitemap: http://www.example.com/sitemap_2.xml
+
 User-Agent: msnbot
 Crawl-Delay: 20
 
@@ -73,15 +76,21 @@
         robotex = Robotex.new
         robotex.delay(SPEC_DOMAIN).should be_nil 
       end
+    end
 
     context 'when Crawl-Delay is specified for the user-agent' do
       it 'returns the delay as a Fixnum' do
         robotex = Robotex.new('msnbot')
         robotex.delay(SPEC_DOMAIN).should == 20
       end
     end
-    end
   end
 
+  describe '#sitemaps' do
+    it 'returns an array of sitemaps' do
+      sitemaps = ['http://www.example.com/sitemap_1.xml','http://www.example.com/sitemap_2.xml']
+      robotex = Robotex.new
+      robotex.sitemaps(SPEC_DOMAIN).should == sitemaps
+    end
+  end
 end
-