# # robots.txt database of http://traumfahne.de # # suggestions, comments, complaints and more infos: http://traumfahne.de/vom-richtigen-umgang-mit-der-robots-txt/ # # if you want it, use copy + paste ;) http://creativecommons.org/licenses/by-sa/3.0/ # User-agent: Googlebot-Image #url: http://www.google.com/ #note: google image search Disallow: / User-agent: MSNBot #url: http://www.msn.com/ #note: msn also accepts a delay Disallow: Crawl-delay: 20 User-agent: Slurp #url: http://help.yahoo.com/help/us/ysearch/slurp/index.html #note: inktomi's "slurp" can read a minimum delay between hits Disallow: Crawl-delay: 20 User-agent: msnbot/1.0 (+http://search.msn.com/msnbot.htm) #url: http://search.msn.com/msnbot.htm #note: msnbots have changed their name recently Disallow: Crawl-delay: 20 User-agent: * #url: http://www.w3.org/TR/html4/appendix/notes.html#h-B.4.1.1 #note: wildcard for the rest of the obeyers. Disallow: #about the ads.txt User-agent: * Disallow: /ads.txt