# # robots.txt database of http://xlqr.de # # suggestions, comments, complaints and more infos: http://dereinzige.de/2007/03/26/robots-database/ # # if you want it, use copy + paste ;) http://creativecommons.org/licenses/by-sa/3.0/ # User-agent: Googlebot/2.1 #url: http://www.google.com/ #note: another name for the same google thing ;) Disallow: User-agent: msnbot-media/1.0 (+http://search.msn.com/msnbot.htm) #url: http://search.msn.com/msnbot.htm #note: msnbots have changed their name recently Disallow: Crawl-delay: 20 User-agent: Googlebot #url: http://www.google.com/support/webmasters/bin/topic.py?topic=8843 #note: the one and only ... webs most used search engine Disallow: User-agent: Googlebot-Image #url: http://www.google.com/ #note: google image search Disallow: User-agent: Mediapartners-Google* #url: http://www.google.com/ #note: googles adwords campain Disallow: User-agent: Mediapartners-Google/2.1 #url: http://www.google.com/ #note: adsense and more Disallow: User-agent: MSNBot #url: http://www.msn.com/ #note: msn also accepts a delay Disallow: Crawl-delay: 20 User-agent: msnbot/1.0 (+http://search.msn.com/msnbot.htm) #url: http://search.msn.com/msnbot.htm #note: msnbots have changed their name recently Disallow: Crawl-delay: 20 User-agent: seekbot #url: http://www.seekport.de/help/webmaster_robot.html #note: young european search engine Disallow: User-agent: Slurp #url: http://help.yahoo.com/help/us/ysearch/slurp/index.html #note: inktomi's "slurp" can read a minimum delay between hits Disallow: Crawl-delay: 20 User-agent: Technoratibot/0.7 #url: #note: Disallow: User-agent: WordPress/2.1.2 #url: #note: wordpress stats Disallow: User-agent: Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/2.2 #url: http://www.seekbot.net/bot.html #note: Disallow: User-agent: Yahoo! Slurp #url: #note: Disallow: User-agent: * #url: http://www.w3.org/TR/html4/appendix/notes.html#h-B.4.1.1 #note: wildcard for the rest of the obeyers. you don't need to add the adjacent following bots ... i did it for the sake of knowledge ;) Disallow: / User-agent: A-Online Search #url: #note: Disallow: / User-agent: aconon Index #url: #note: Disallow: / User-agent: ActiveAgent #url: #note: Disallow: / User-agent: Alexibot #url: #note: Disallow: / User-agent: Aqua_Products #url: #note: Disallow: / User-agent: Arachnoidea #url: #note: Disallow: / User-agent: Arachnophilia #url: #note: Disallow: / User-agent: ArchitextSpider #url: #note: Disallow: / User-agent: AskJeeves #url: #note: Disallow: / User-agent: asterias #url: #note: Disallow: / User-agent: ATN_Worldwide #url: #note: Disallow: / User-agent: autoemailspider #url: #note: Disallow: / User-agent: b2w #url: #note: Disallow: / User-agent: b2w 0.1 #url: #note: Disallow: / User-agent: b2w/0.1 #url: #note: Disallow: / User-agent: BackDoorBot #url: #note: Disallow: / User-agent: BackDoorBot 1.0 #url: #note: Disallow: / User-agent: BackDoorBot/1.0 #url: #note: Disallow: / User-agent: BackWeb #url: #note: Disallow: / User-agent: baiduspider #url: #note: Disallow: / User-agent: BecomeBot #url: #note: Disallow: / User-agent: Black Hole #url: #note: Disallow: / User-agent: BlackWidow #url: #note: Disallow: / User-agent: BlowFish #url: #note: Disallow: / User-agent: BlowFish 1.0 #url: #note: Disallow: / User-agent: BlowFish/1.0 #url: #note: Disallow: / User-agent: boitho.com-dc #url: #note: Disallow: / User-agent: Bookmark search tool #url: #note: Disallow: / User-agent: BotALot #url: #note: Disallow: / User-agent: BotRightHere #url: #note: Disallow: / User-agent: BuiltBotTough #url: #note: Disallow: / User-agent: Bullseye #url: #note: Disallow: / User-agent: Bullseye/1.0 #url: #note: Disallow: / User-agent: BunnySlippers #url: #note: Disallow: / User-agent: Cegbfeieh #url: #note: Disallow: / User-agent: CheeseBot #url: #note: Disallow: / User-agent: CherryPicker #url: #note: Disallow: / User-agent: CherryPicker /1.0 #url: #note: Disallow: / User-agent: CherryPicker 1.0 #url: #note: Disallow: / User-agent: CherryPickerElite 1.0 #url: #note: Disallow: / User-agent: CherryPickerElite/1.0 #url: #note: Disallow: / User-agent: CherryPickerSE 1.0 #url: #note: Disallow: / User-agent: CherryPickerSE/1.0 #url: #note: Disallow: / User-agent: ChinaClaw #url: #note: Disallow: / User-agent: Collector #url: #note: Disallow: / User-agent: CoolBot #url: #note: Disallow: / User-agent: Copernic #url: #note: Disallow: / User-agent: Copier #url: #note: Disallow: / User-agent: CopyRightCheck #url: #note: Disallow: / User-agent: cosmos #url: #note: Disallow: / User-agent: Crawler V 0.2.1 admin@crawler.de #url: #note: Disallow: / User-agent: Crescent #url: #note: Disallow: / User-agent: Crescent Internet ToolPak HTTPOLE Control v.1.0 #url: #note: Disallow: / User-agent: Deweb/1.01 #url: #note: Disallow: / User-agent: DISCo #url: #note: Disallow: / User-agent: DISCo Pump #url: #note: Disallow: / User-agent: DISCo Pump 3.1 #url: #note: Disallow: / User-agent: DittoSpyder #url: #note: Disallow: / User-agent: dloader(naverrobot)/1.0 #url: #note: Disallow: / User-agent: DOC #url: #note: Disallow: / User-agent: Download Demon #url: #note: Disallow: / User-agent: Download Ninja #url: #note: Disallow: / User-agent: Download Wonder #url: #note: Disallow: / User-agent: Downloader #url: #note: Disallow: / User-agent: Drip #url: #note: Disallow: / User-agent: dumbot #url: #note: Disallow: / User-agent: eCatch #url: #note: Disallow: / User-agent: EirGrabber #url: #note: Disallow: / User-agent: emailcollector #url: #note: Disallow: / User-agent: EmailCollector #url: #note: Disallow: / User-agent: EmailCollector 1.0 #url: #note: Disallow: / User-agent: EmailSiphon #url: #note: Disallow: / User-agent: EmailWolf #url: #note: Disallow: / User-agent: EmailWolf 1.00 #url: #note: Disallow: / User-agent: Enterprise_Search #url: #note: Disallow: / User-agent: Enterprise_Search/1.0 #url: #note: Disallow: / User-agent: EroCrawler #url: #note: Disallow: / User-agent: es #url: #note: Disallow: / User-agent: Eule-Robot #url: #note: Disallow: / User-agent: Express WebPictures #url: #note: Disallow: / User-agent: ExtractorPro #url: #note: Disallow: / User-agent: EyeNetIE #url: #note: Disallow: / User-agent: FairAd Client #url: #note: Disallow: / User-agent: FAST #url: #note: Disallow: / User-agent: FAST Crawler (crawler at fast.no) #url: #note: Disallow: / User-agent: Fetch #url: #note: Disallow: / User-agent: fido/1.0 #url: #note: Disallow: / User-agent: FileHound #url: #note: Disallow: / User-agent: Firefly/1.0 #url: #note: Disallow: / User-agent: Flaming AttackBot #url: #note: Disallow: / User-agent: FlashGet #url: #note: Disallow: / User-agent: Foobot #url: #note: Disallow: / User-agent: FreeFind #url: #note: Disallow: / User-agent: Gaisbot #url: #note: Disallow: / User-agent: GetRight #url: #note: Disallow: / User-agent: GetRight/4.2 #url: #note: Disallow: / User-agent: GetSmart #url: #note: Disallow: / User-agent: GetWeb! #url: #note: Disallow: / User-agent: Gigabot #url: #note: Disallow: / User-agent: Go-Ahead-Got-It #url: #note: Disallow: / User-agent: Go!Zilla #url: #note: Disallow: / User-agent: gotit #url: #note: Disallow: / User-agent: Grabber #url: #note: Disallow: / User-agent: grabber #url: #note: Disallow: / User-agent: GrabNet #url: #note: Disallow: / User-agent: Grafula #url: #note: Disallow: / User-agent: grub #url: #note: Disallow: / User-agent: grub-client #url: #note: Disallow: / User-agent: Gulliver/1.3 #url: #note: Disallow: / User-agent: Harvest #url: #note: Disallow: / User-agent: Harvest 1.5 #url: #note: Disallow: / User-agent: Harvest/1.4.pl2 #url: #note: Disallow: / User-agent: Harvest/1.5 #url: #note: Disallow: / User-agent: Hatena Antenna #url: #note: Disallow: / User-agent: HenryTheMiragoRobot #url: #note: Disallow: / User-agent: hloader #url: #note: Disallow: / User-agent: HLoader #url: #note: Disallow: / User-agent: HMView #url: #note: Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler #url: #note: Disallow: / User-agent: httplib #url: #note: Disallow: / User-agent: HTTrack #url: #note: Disallow: / User-agent: humanlinks #url: #note: Disallow: / User-agent: ia_archiver #url: #note: way back machine Disallow: / User-agent: ia_archiver/1.6 #url: #note: Disallow: / User-agent: IBM_Planetwide #url: #note: Disallow: / User-agent: Image Stripper #url: #note: Disallow: / User-agent: Image Sucker #url: #note: Disallow: / User-agent: Indy Library #url: #note: Disallow: / User-agent: InfoNaviRobot #url: #note: Disallow: / User-agent: InfoSeek Sidewinder/0.9 #url: #note: Disallow: / User-agent: InterGET #url: #note: Disallow: / User-agent: Internet Ninja #url: #note: Disallow: / User-agent: Iria #url: #note: Disallow: / User-agent: IRLbot #url: http://irl.cs.tamu.edu/crawler/ #note: IRL-crawler is a Texas A+M research project Disallow: / Crawl-delay: 20 User-agent: Iron33 #url: #note: Disallow: / User-agent: Iron33/1.0.2 #url: #note: Disallow: / User-agent: Jeeves #url: #note: Disallow: / User-agent: JennyBot #url: #note: Disallow: / User-agent: Jetbot #url: #note: Disallow: / User-agent: Jetbot/1.0 #url: #note: Disallow: / User-agent: JetCar #url: #note: Disallow: / User-agent: JOC #url: #note: Disallow: / User-agent: JOC Web Spider #url: #note: Disallow: / User-agent: JustView #url: #note: Disallow: / User-agent: k2spider #url: #note: Disallow: / User-agent: Kenjin Spider #url: #note: Disallow: / User-agent: Keyword Density #url: #note: Disallow: / User-agent: Keyword Density/0.9 #url: #note: Disallow: / User-agent: KIT-Fireball/2.0 #url: #note: Disallow: / User-agent: larbin #url: #note: Disallow: / User-agent: larbin #url: #note: Disallow: / User-agent: LexiBot #url: #note: Disallow: / User-agent: lftp #url: #note: Disallow: / User-agent: libWeb #url: #note: Disallow: / User-agent: libWeb/clsHTTP #url: #note: Disallow: / User-agent: libwww #url: #note: Disallow: / User-agent: likse #url: #note: Disallow: / User-agent: LinkextractorPro #url: #note: Disallow: / User-agent: linko #url: #note: Disallow: / User-agent: LinkScan #url: #note: Disallow: / User-agent: LinkScan/8.1a Unix #url: #note: Disallow: / User-agent: LinkWalker #url: #note: Disallow: / User-agent: LNSpiderguy #url: #note: Disallow: / User-agent: looksmart #url: #note: Disallow: / User-agent: lwp-trivial #url: #note: Disallow: / User-agent: lwp-trivial/1.34 #url: #note: Disallow: / User-agent: Lycos_Spider_(T-Rex) #url: #note: Disallow: / User-agent: Mag-Net #url: #note: Disallow: / User-agent: Magnet #url: #note: Disallow: / User-agent: Mass Downloader #url: #note: Disallow: / User-agent: Mata Hari #url: #note: Disallow: / User-agent: MegaSheep #url: #note: Disallow: / User-agent: Memo #url: #note: Disallow: / User-agent: Microsoft URL Control #url: #note: Disallow: / User-agent: Microsoft URL Control - 5.01.4511 #url: #note: Disallow: / User-agent: Microsoft URL Control - 6.00.8169 #url: #note: Disallow: / User-agent: Microsoft.URL.Control #url: #note: Disallow: / User-agent: MIDown tool #url: #note: Disallow: / User-agent: MIIxpc #url: #note: Disallow: / User-agent: MIIxpc/4.2 #url: #note: Disallow: / User-agent: Mirror #url: #note: Disallow: / User-agent: Mister PiX #url: #note: Disallow: / User-agent: moget #url: #note: Disallow: / User-agent: moget/2.1 #url: #note: Disallow: / User-agent: mozilla #url: #note: Disallow: / User-agent: Mozilla #url: #note: Disallow: / User-agent: mozilla/3 #url: #note: Disallow: / User-agent: mozilla/4 #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows ME) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP) #url: #note: Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; AIRF) #url: #note: Disallow: / User-agent: mozilla/5 #url: #note: Disallow: / User-agent: MSIECrawler #url: #note: Disallow: / User-agent: my-heritrix-crawler(+http://mywebsite.com) #url: http://mywebsite.com #note: Disallow: / User-agent: naver #url: #note: Disallow: / User-agent: Navroad #url: #note: Disallow: / User-agent: NearSite #url: #note: Disallow: / User-agent: Net Vampire #url: #note: Disallow: / User-agent: NetAnts #url: #note: Disallow: / User-agent: NetMechanic #url: #note: Disallow: / User-agent: NetSpider #url: #note: Disallow: / User-agent: NetZIP #url: #note: Disallow: / User-agent: NICErsPRO #url: #note: Disallow: / User-agent: Ninja #url: #note: Disallow: / User-agent: NPBot #url: #note: Disallow: / User-agent: Nutch #url: #note: Disallow: / User-agent: Octopus #url: #note: Disallow: / User-agent: Offline Explorer #url: #note: Disallow: / User-agent: Offline Navigator #url: #note: Disallow: / User-agent: OmniExplorer_Bot #url: #note: Disallow: / User-agent: Openbot #url: #note: Disallow: / User-agent: Openfind #url: #note: Disallow: / User-agent: Openfind data gathere #url: #note: Disallow: / User-agent: Openfind data gatherer #url: #note: Disallow: / User-agent: Oracle Ultra Search #url: #note: Disallow: / User-agent: PageGrabber #url: #note: Disallow: / User-agent: Papa Foto #url: #note: Disallow: / User-agent: pavuk #url: #note: Disallow: / User-agent: pcBrowser #url: #note: Disallow: / User-agent: PerMan #url: #note: Disallow: / User-agent: ProPowerBot #url: #note: Disallow: / User-agent: ProPowerBot/2.14 #url: #note: Disallow: / User-agent: ProWebWalker #url: #note: Disallow: / User-agent: psbot #url: #note: Disallow: / User-agent: Pump #url: #note: Disallow: / User-agent: Python-urllib #url: #note: Disallow: / User-agent: QueryN Metasearch #url: #note: Disallow: / User-agent: Radiation #url: #note: Disallow: / User-agent: Radiation Retriever #url: #note: Disallow: / User-agent: Radiation Retriever 1.1 #url: #note: Disallow: / User-agent: RealDownload #url: #note: Disallow: / User-agent: Reaper #url: #note: Disallow: / User-agent: Recorder #url: #note: Disallow: / User-agent: ReGet #url: #note: Disallow: / User-agent: RepoMonkey #url: #note: Disallow: / User-agent: RepoMonkey Bait Tackle/v1.01 #url: #note: Disallow: / User-agent: RMA #url: #note: Disallow: / User-agent: Robozilla #url: #note: Disallow: / User-agent: Roverbot #url: #note: Disallow: / User-agent: scooter #url: #note: Disallow: / User-agent: Scooter #url: #note: Disallow: / User-agent: Scrubby #url: #note: Disallow: / User-agent: searchpreview #url: #note: Disallow: / User-agent: Siphon #url: #note: Disallow: / User-agent: sitecheck.internetseer.com #url: #note: Disallow: / User-agent: SiteSnagger #url: #note: Disallow: / User-agent: SmartDownload #url: #note: Disallow: / User-agent: Snake #url: #note: Disallow: / User-agent: sootle #url: #note: Disallow: / User-agent: SpaceBison #url: #note: Disallow: / User-agent: SpankBot #url: #note: Disallow: / User-agent: spanner #url: #note: Disallow: / User-agent: Stanford #url: #note: Disallow: / User-agent: Stanford Comp Sci #url: #note: Disallow: / User-agent: Sucker #url: #note: Disallow: / User-agent: SuperBot #url: #note: Disallow: / User-agent: SuperHTTP #url: #note: Disallow: / User-agent: Surfbot #url: #note: Disallow: / User-agent: suzuran #url: #note: Disallow: / User-agent: Szukacz #url: #note: Disallow: / User-agent: Szukacz/1.4 #url: #note: Disallow: / User-agent: tAkeOut #url: #note: Disallow: / User-agent: Tarantula #url: #note: Disallow: / User-agent: Teleport #url: #note: Disallow: / User-agent: Teleport Pro #url: #note: Disallow: / User-agent: Teleport Pro/1.29.1590 #url: #note: Disallow: / User-agent: Teleport Pro/1.29.1616 #url: #note: Disallow: / User-agent: Teleport Pro/1.29.1632 #url: #note: Disallow: / User-agent: Teleport Pro/1.29.1718 #url: #note: Disallow: / User-agent: TeleportPro #url: #note: Disallow: / User-agent: TeleportPro #url: #note: Disallow: / User-agent: Telesoft #url: #note: Disallow: / User-agent: Teoma #url: #note: Disallow: / User-agent: The Intraformant #url: #note: Disallow: / User-agent: TheNomad #url: #note: Disallow: / User-agent: thesubot #url: #note: Disallow: / User-agent: TightTwatBot #url: #note: Disallow: / User-agent: Titan #url: #note: Disallow: / User-agent: toCrawl/UrlDispatcher #url: #note: Disallow: / User-agent: True_Robot #url: #note: Disallow: / User-agent: True_Robot/1.0 #url: #note: Disallow: / User-agent: turingos #url: #note: Disallow: / User-agent: UbiCrawler #url: #note: Disallow: / User-agent: Ultraseek #url: #note: Disallow: / User-agent: URL Control #url: #note: Disallow: / User-agent: URL_Spider_Pro #url: #note: Disallow: / User-agent: URLy Warning #url: #note: Disallow: / User-agent: Vacuum #url: #note: Disallow: / User-agent: VCI #url: #note: Disallow: / User-agent: VCI WebViewer VCI WebViewer Win32 #url: #note: Disallow: / User-agent: VoidEYE #url: #note: Disallow: / User-agent: Web Image Collector #url: #note: Disallow: / User-agent: Web Sucker #url: #note: Disallow: / User-agent: WebAuto #url: #note: Disallow: / User-agent: WebBandit #url: #note: Disallow: / User-agent: WebBandit 2.1 #url: #note: Disallow: / User-agent: WebBandit 3.50 #url: #note: Disallow: / User-agent: webbandit 4.00.0 #url: #note: Disallow: / User-agent: WebBandit/3.50 #url: #note: Disallow: / User-agent: WebCapture 2.0 #url: #note: Disallow: / User-agent: WebCopier #url: #note: Disallow: / User-agent: WebCopier v.2.2 #url: #note: Disallow: / User-agent: WebCopier v3.2a #url: #note: Disallow: / User-agent: WebCrawler_1.1 internet@bredband.net #url: http://www.bredband.net/ #note: a swedish provider Disallow: / User-agent: WebEMailExtrac. #url: #note: Disallow: / User-agent: WebEMailExtractor 1.0B #url: #note: Disallow: / User-agent: WebEnhancer #url: #note: Disallow: / User-agent: WebFetch #url: #note: Disallow: / User-agent: WebGo IS #url: #note: Disallow: / User-agent: WebLeacher #url: #note: Disallow: / User-agent: WebmasterWorld #url: #note: Disallow: / User-agent: WebmasterWorld Extractor #url: #note: Disallow: / User-agent: WebmasterWorldForumBot #url: #note: Disallow: / User-agent: WebMoose #url: #note: Disallow: / User-agent: WebReaper #url: #note: Disallow: / User-agent: WebSauger #url: #note: Disallow: / User-agent: Website #url: #note: Disallow: / User-agent: Website eXtractor #url: #note: Disallow: / User-agent: Website Quester #url: #note: Disallow: / User-agent: Webster #url: #note: Disallow: / User-agent: Webster Pro #url: #note: Disallow: / User-agent: WebStripper #url: #note: Disallow: / User-agent: WebVac #url: #note: Disallow: / User-agent: WebWhacker #url: #note: Disallow: / User-agent: WebZip #url: #note: Disallow: / User-agent: WebZip #url: #note: Disallow: / User-agent: WebZip/4.0 #url: #note: Disallow: / User-agent: WebZIP/4.21 #url: #note: Disallow: / User-agent: WebZIP/5.0 #url: #note: Disallow: / User-agent: wget #url: #note: Disallow: / User-agent: Wget #url: #note: Disallow: / User-agent: Wget/1.5.3 #url: #note: Disallow: / User-agent: Wget/1.6 #url: #note: Disallow: / User-agent: Whacker #url: #note: Disallow: / User-agent: whatUseek_winona #url: #note: Disallow: / User-agent: WhoWhere #url: #note: Disallow: / User-agent: Widow #url: #note: Disallow: / User-agent: WWW-Collector #url: #note: Disallow: / User-agent: WWW-Collector-E #url: #note: Disallow: / User-agent: WWWOFFLE #url: #note: Disallow: / User-agent: WX_mail #url: #note: Disallow: / User-agent: Xaldon #url: #note: Disallow: / User-agent: Xaldon/WebSpider #url: #note: Disallow: / User-agent: Xenu #url: #note: Disallow: / User-agent: Xenu's #url: #note: Disallow: / User-agent: Xenu's Link Sleuth 1.1c #url: #note: Disallow: / User-agent: Xenu\'s #url: #note: Disallow: / User-agent: Xenu\'s Link Sleuth 1.1c #url: #note: Disallow: / User-agent: yahoo-blogs/v3.9 #url: #note: Disallow: / User-agent: yahoo-mmcrawler #url: #note: Disallow: / User-agent: Zao #url: #note: Disallow: / User-agent: Zealbot #url: #note: Disallow: / User-agent: Zeus #url: #note: Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 #url: #note: Disallow: / User-agent: Zeus Link Scout #url: #note: Disallow: / User-agent: ZyBORG #url: #note: Disallow: / User-agent: ichiro/2.0 (http://help.goo.ne.jp/door/crawler.html) #url: http://help.goo.ne.jp/door/crawler.html #note: Disallow: / User-agent: TMCrawler #url: #note: Disallow: / # # top # # have a nice day :) #