Files
lib/spider.rb
lib/spider/included_in_memcached.rb
lib/spider/next_urls_in_sqs.rb
lib/spider/robot_rules.rb
lib/spider/spider_instance.rb
spec/spec_helper.rb
spec/spider/included_in_memcached_spec.rb
spec/spider/spider_instance_spec.rb
spec/spider_spec.rb