# # Robots.txt file created by http://www.dhpro.com # For domain: http://www.dhpro.com # (C) 2007 # Please obey robots.txt. # All robots will spider the domain # Example User-agent: FreeFind # Example Disallow: / # Will block agent: spiders / bots name & Disallow: / means index nothing # Check your robots or robot @ http://tool.motoricerca.info/robots-checker.phtml User-agent: blackwidow Disallow: / User-agent: bot\ .ru Disallow: / User-agent: bot\ mailto:craftbot@yahoo.com Disallow: / User-agent: cherrypicker Disallow: / User-agent: chinaclaw Disallow: / User-agent: crescent Disallow: / User-agent: custo Disallow: / User-agent: disco Disallow: / User-agent: download\ demon Disallow: / User-agent: download ninja Disallow: / User-agent: ecatch Disallow: / User-agent: eirgrabber Disallow: / User-agent: emailcollector Disallow: / User-agent: emailsiphon Disallow: / User-agent: emailwolf Disallow: / User-agent: express\ webpictures Disallow: / User-agent: extractorpro Disallow: / User-agent: eyenetie Disallow: / User-agent: flashget Disallow: / User-agent: funwebproducts Disallow: / User-agent: getright Disallow: / User-agent: getbot Disallow: / User-agent: getweb! Disallow: / User-agent: go!zilla Disallow: / User-agent: go-ahead-got-it Disallow: / User-agent: gornker Disallow: / User-agent: grabnet Disallow: / User-agent: grafula Disallow: / # # the 'grub' distributed client has been *very* poorly behaved. # User-agent: grub-client Disallow: / # User-agent: fetch # Disallow: / User-agent: hmview Disallow: / User-agent: httrack Disallow: / # don't allow the wayback-maschine to index User-pages User-agent: ia_archiver Disallow: / User-agent: image\ stripper Disallow: / User-agent: image\ sucker Disallow: / User-agent: indy\ library Disallow: / User-agent: interget Disallow: / User-agent: internet\ ninja Disallow: / User-agent: infopath.1 Disallow: / User-agent: irvine Disallow: / User-agent: jetcar Disallow: / User-agent: joc\ web\ spider Disallow: / # # doesn't follow robots.txt anyway, but... # User-agent: k2spider Disallow: / User-agent: larbin Disallow: / User-agent: leechftp Disallow: / User-agent: libwww Disallow: / User-agent: linko Disallow: / User-agent: mass\ downloader Disallow: / #User-agent: microsoft.url.control #Disallow: / User-agent: midown\ tool Disallow: / User-agent: mister\ pix Disallow: / User-agent: msiecrawler Disallow: / User-agent: mozilla.*newt Disallow: / User-agent: nasa search 1.0 Disallow: / User-agent: navroad Disallow: / User-agent: nearsite Disallow: / User-agent: netants Disallow: / User-agent: netspider Disallow: / User-agent: net\ vampire Disallow: / User-agent: netzip Disallow: / User-agent: nicerspro Disallow: / # # hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: npbot Disallow: / User-agent: octopus Disallow: / User-agent: offline\ explorer Disallow: / User-agent: offline\ navigator Disallow: / User-agent: omniexplorer_bot Disallow: / User-agent: pagegrabber Disallow: / User-agent: papa\ foto Disallow: / User-agent: pavuk Disallow: / User-agent: pcbrowser Disallow: / User-agent: dloader(naverrobot) Disallow: / #User-agent: puf #Disallow: / #User-agent: realdownload #Disallow: / User-agent: reget Disallow: / User-agent: searchexpress Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: sitesnagger Disallow: / User-agent: smartdownload Disallow: / User-agent: sogou spider Disallow: / User-agent: sohu agent Disallow: / User-agent: superbot Disallow: / User-agent: superhttp Disallow: / User-agent: surfbot Disallow: / User-agent: siphon Disallow: / User-agent: takeout Disallow: / User-agent: teleport Disallow: / User-agent: teleport\ pro Disallow: / User-agent: voideye Disallow: / User-agent: web\ image\ collector Disallow: / User-agent: web\ sucker Disallow: / User-agent: webcopier Disallow: / User-agent: webauto Disallow: / User-agent: webbandit Disallow: / User-agent: webgo\ is Disallow: / User-agent: webfetch Disallow: / User-agent: webleacher Disallow: / User-agent: webreaper Disallow: / User-agent: websauger Disallow: / User-agent: webstripper Disallow: / User-agent: webwhacker Disallow: / User-agent: website\ extractor Disallow: / User-agent: website\ quester Disallow: / # a capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: webzip Disallow: / #User-agent: wget #Disallow: / User-agent: widow Disallow: / User-agent: wwwoffle Disallow: / User-agent: xaldon\ webspider Disallow: / User-agent: xenu Disallow: / User-agent: zealbot Disallow: / User-agent: zeus Disallow: / User-agent: zyborg Disallow: / User-agent: wget Disallow: / User-agent: * Disallow: /aff/ Disallow: /aff/special/ Disallow: /banners01/ Disallow: /cgi/ Disallow: /DNDLGC/ Disallow: /hb2u/ Disallow: /myevent/