User-agent: Scrapy Allow: / User-Agent: * Allow: /?display=wide Allow: /salvationarmy/wp-content/uploads/ Disallow: /salvationarmy/wp-content/plugins/ Disallow: /salvationarmy/readme.html Disallow: /salvationarmy/refer/ Sitemap: http://salvationarmy.org.za/sitemap_index.xml Sitemap: http://salvationarmy.org.za/post-sitemap.xml Sitemap: http://salvationarmy.org.za/page-sitemap.xml Sitemap: http://salvationarmy.org.za/category-sitemap.xml