# robots.txt for http://www.dstl.gov.uk # # No bots, except crawlers for public search indexes: # Exalead, Gigablast, Google, Msn, Yahoo/Slurp, Ask/Teoma, Cuil, Voila # # # Sitemap: User-agent: Exabot Disallow: /systemsskills/ User-agent: Gigabot Disallow: /systemsskills/ User-agent: Googlebot Disallow: /systemsskills/ User-agent: Mediapartners-Google Disallow: /systemsskills/ User-agent: msnbot Disallow: /systemsskills/ User-agent: Slurp Disallow: /systemsskills/ User-agent: Teoma Disallow: /systemsskills/ User-agent: Twiceler Disallow: /systemsskills/ User-agent: VoilaBot Disallow: /systemsskills/ User-agent: yacybot Disallow: /systemsskills/ User-agent: Exabot Disallow: /nexus/ User-agent: Gigabot Disallow: /nexus/ User-agent: Googlebot Disallow: /nexus/ User-agent: Mediapartners-Google Disallow: /nexus/ User-agent: msnbot Disallow: /nexus/ User-agent: Slurp Disallow: /nexus/ User-agent: Teoma Disallow: /nexus/ User-agent: Twiceler Disallow: /nexus/ User-agent: VoilaBot Disallow: /nexus/ User-agent: yacybot Disallow: /nexus/ User-agent: Exabot Disallow: /finex/ User-agent: Gigabot Disallow: /finex/ User-agent: Googlebot Disallow: /finex/ User-agent: Mediapartners-Google Disallow: /finex/ User-agent: msnbot Disallow: /finex/ User-agent: Slurp Disallow: /finex/ User-agent: Teoma Disallow: /finex/ User-agent: Twiceler Disallow: /finex/ User-agent: VoilaBot Disallow: /finex/ User-agent: yacybot Disallow: /finex/ User-agent: Exabot Disallow: /JobApp/ User-agent: Gigabot Disallow: /JobApp/ User-agent: Googlebot Disallow: /JobApp/ User-agent: Mediapartners-Google Disallow: /JobApp/ User-agent: msnbot Disallow: /JobApp/ User-agent: Slurp Disallow: /JobApp/ User-agent: Teoma Disallow: /JobApp/ User-agent: Twiceler Disallow: /JobApp/ User-agent: VoilaBot Disallow: /JobApp/ User-agent: yacybot Disallow: /JobApp/ User-agent: Exabot Disallow: /onthemove/ User-agent: Gigabot Disallow: /onthemove/ User-agent: Googlebot Disallow: /onthemove/ User-agent: Mediapartners-Google Disallow: /onthemove/ User-agent: msnbot Disallow: /onthemove/ User-agent: Slurp Disallow: /onthemove/ User-agent: Teoma Disallow: /onthemove/ User-agent: Twiceler Disallow: /onthemove/ User-agent: VoilaBot Disallow: /onthemove/ User-agent: yacybot Disallow: /onthemove/ User-agent: Exabot Disallow: /cgi-bin/perlfect/search/ User-agent: Gigabot Disallow: /cgi-bin/perlfect/search/ User-agent: Googlebot Disallow: /cgi-bin/perlfect/search/ User-agent: Mediapartners-Google Disallow: /cgi-bin/perlfect/search/ User-agent: msnbot Disallow: /cgi-bin/perlfect/search/ User-agent: Slurp Disallow: /cgi-bin/perlfect/search/ User-agent: Teoma Disallow: /cgi-bin/perlfect/search/ User-agent: Twiceler Disallow: /cgi-bin/perlfect/search/ User-agent: VoilaBot Disallow: /cgi-bin/perlfect/search/ User-agent: yacybot Disallow: /cgi-bin/perlfect/search/ User-agent: Exabot Disallow: /cgi-bin/ User-agent: Gigabot Disallow: /cgi-bin/ User-agent: Googlebot Disallow: /cgi-bin/ User-agent: Mediapartners-Google Disallow: /cgi-bin/ User-agent: msnbot Disallow: /cgi-bin/ User-agent: Slurp Disallow: /cgi-bin/ User-agent: Teoma Disallow: /cgi-bin/ User-agent: Twiceler Disallow: /cgi-bin/ User-agent: VoilaBot Disallow: /cgi-bin/ User-agent: yacybot Disallow: /cgi-bin/ User-agent: Exabot Disallow: /robots.txt/ User-agent: Gigabot Disallow: /robots.txt/ User-agent: Googlebot Disallow: /robots.txt/ User-agent: Mediapartners-Google Disallow: /robots.txt/ User-agent: msnbot Disallow: /robots.txt/ User-agent: Slurp Disallow: /robots.txt/ User-agent: Teoma Disallow: /robots.txt/ User-agent: Twiceler Disallow: /robots.txt/ User-agent: VoilaBot Disallow: /robots.txt/ User-agent: yacybot Disallow: /robots.txt/ #