User-agent: panscient.com Disallow: / User-agent: vscooter Disallow: / User-agent: psbot Disallow: / User-agent: ia_archiver Disallow: / User-agent: MJ12bot Disallow: / User-agent: twiceler Disallow: / User-agent: Yandex Disallow: / User-agent: taptubot Disallow: / User-agent: Googlebot-Image Disallow: / User-agent: Facebot Allow: * User-agent: Twitterbot Allow: * User-agent: TwengaBot Disallow: / User-agent: sitebot Disallow: / User-agent: Baiduspider Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Ezooms Disallow: / User-agent: sistrix Disallow: / User-agent: aiHitBot Disallow: / User-agent: InfoPath Disallow: / User-agent: InfoPath.2 Disallow: / User-agent: swebot Disallow: / User-agent: EC2LinkFinder Disallow: / User-agent: TurnitinBot Disallow: / User-agent: SearchmetericsBot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Exabot Disallow: / User-agent: Sosospider Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: netEstate NE Crawler Disallow: / User-agent: Aboundexbot Disallow: / User-agent: Aboundex Disallow: / User-agent: meanpathbot Disallow: / User-agent: Mail.Ru Disallow: / User-agent: spbot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: LinkpadBot Disallow: / User-agent: EasouSpider Disallow: / User-agent: SeznamBot Disallow: / User-agent: wotbox Disallow: / User-agent: BLEXBot Disallow: / User-agent: XoviBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: A6-Indexer Disallow: / User-agent: Riddler Disallow: / User-agent: LoadTimeBot Disallow: / User-agent: oBot Disallow: / User-agent: MojeekBot Disallow: / User-agent: memoryBot Disallow: / User-agent: ltx71 Disallow: / User-agent: AdvBot Disallow: / User-agent: SMTBot Disallow: / User-agent: YisouSpider Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: gsa-crawler Disallow: / User-agent: Nutch Disallow: / User-agent: tbot-nutch Disallow: / User-agent: thunderstone Disallow: / User-agent: yacybot Disallow: / User-agent: RankSonicBot Disallow: / User-agent: betaBot Disallow: / User-agent: parsijoo-bot Disallow: / User-agent: ltx71 Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: gocrawl Disallow: / User-agent: plukkie Disallow: / User-agent: Applebot Disallow: / User-agent: Lipperhey Disallow: / User-agent: SafeDNSBot Disallow: / User-agent: RankActiveLinkBot Disallow: / User-agent: Sogou web spider Disallow: / User-agent: Uptimebot Disallow: / User-agent: Seeker Disallow: / User-agent: Cliqzbot Disallow: / User-agent: DomainCrawler Disallow: / User-agent: yoozBot Disallow: / User-agent: Slurp Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI Disallow: /*.avi Disallow: /disclaimer.html Disallow: /security.html Disallow: /poweredby.html Disallow: /about_smythies.html Disallow: /unused_link.html Disallow: /old_pages.html Disallow: /index_0* Disallow: /*index_0*$ Disallow: /digital_camera/ Disallow: /lab/ Disallow: /xmas_* Disallow: /~doug/archives/ Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ Disallow: /bot_trap.html User-agent: Googlebot Disallow: /*.jpg$ Disallow: /*.JPG$ Disallow: /*.png$ Disallow: /*.PDF$ Disallow: /*.pdf$ Disallow: /*.mp3$ Disallow: /*.MOV$ Disallow: /*.mov$ Disallow: /*.AVI$ Disallow: /*.avi$ Disallow: /index_0*$ Disallow: /*index_0*$ Disallow: /xmas_* Disallow: /~doug/archives/ Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ Disallow: /~doug/2010.01.23/ Disallow: /~doug/2007.11.20/ Disallow: /~doug/2004.06.26/ Disallow: /digital_camera/ Disallow: /old_pages.html Disallow: /unused_link.html Disallow: /disclaimer.html Disallow: /security.html Disallow: /about_smythies.html Disallow: /poweredby.html Disallow: /DSCN*.htm Disallow: /bot_trap.html User-agent: msnbot Disallow: /*.jpg$ Disallow: /*.JPG$ Disallow: /*.png$ Disallow: /*.PDF$ Disallow: /*.pdf$ Disallow: /*.mp3$ Disallow: /*.MOV$ Disallow: /*.mov$ Disallow: /*.AVI$ Disallow: /*.avi$ Disallow: /disclaimer.html Disallow: /security.html Disallow: /poweredby.html Disallow: /about_smythies.html Disallow: /unused_link.html Disallow: /old_pages.html Disallow: /index_0* Disallow: /*index_0*$ Disallow: /digital_camera/ Disallow: /lab/ Disallow: /xmas_* Disallow: /~doug/archives/ Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ Disallow: /bot_trap.html User-agent: bingbot Disallow: / Crawl-delay: 1800 # Disallow: /*.jpg$ # Disallow: /*.JPG$ # Disallow: /*.png$ # Disallow: /*.PDF$ # Disallow: /*.pdf$ # Disallow: /*.mp3$ # Disallow: /*.MOV$ # Disallow: /*.mov$ # Disallow: /*.AVI$ # Disallow: /*.avi$ # Disallow: /disclaimer.html # Disallow: /security.html # Disallow: /poweredby.html # Disallow: /about_smythies.html # Disallow: /unused_link.html # Disallow: /old_pages.html # Disallow: /index_0* # Disallow: /*index_0*$ # Disallow: /digital_camera/ # Disallow: /lab/ # Disallow: /xmas_* # Disallow: /~doug/archives/ # Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ # Disallow: /bot_trap.html User-agent: Qwantify Crawl-delay: 3600 Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI Disallow: /*.avi Disallow: /disclaimer.html Disallow: /security.html Disallow: /poweredby.html Disallow: /about_smythies.html Disallow: /unused_link.html Disallow: /old_pages.html Disallow: /index_0* Disallow: /*index_0*$ Disallow: /digital_camera/ Disallow: /lab/ Disallow: /xmas_* Disallow: /~doug/archives/ Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ Disallow: /bot_trap.html User-agent: * Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI Disallow: /*.avi Disallow: /disclaimer.html Disallow: /security.html Disallow: /poweredby.html Disallow: /about_smythies.html Disallow: /unused_link.html Disallow: /old_pages.html Disallow: /index_0* Disallow: /*index_0*$ Disallow: /digital_camera/ Disallow: /lab/ Disallow: /xmas_* Disallow: /~doug/archives/ Disallow: /~doug/linux/ubuntu-docs/help.ubuntu.com/ Disallow: /bot_trap.html Disallow: / Crawl-delay: 1800