# robots.txt for http://www.kolta.net/ # # http://www.robotstxt.org/wc/exclusion.html#robotstxt # # Note that you need a separate "Disallow" line for every URL prefix to exclude # -- you cannot say "Disallow: /cgi-bin/ /tmp/". # Also, you may not have blank lines in a record, # as they are used to delimit multiple records. # Lines containing only a comment are discarded completely, # and therefore do not indicate a record boundary. ## http://www.singingfish.com/spider.html indexes media files: # User-agent: asterias ## http://www.alexa.com/support/for_webmasters.html # User-agent: ia_archiver User-Agent: NPBot Disallow: / User-Agent: sitecheck.internetseer.com Disallow: /file-to-disallow # 20041030: # http://www.whois.sc/" "SurveyBot/2.3 (Whois Source)" User-Agent: SurveyBot Disallow: / # 20040404: User-agent: Uptimebot Disallow: / User-agent: * Disallow: /_vti_bin/ Disallow: /_vti_cnf/ Disallow: /_vti_log/ Disallow: /_vti_pvt/ Disallow: /_vti_txt/ Disallow: /cgi/ Disallow: /cgi-bin/ Disallow: /conf/ Disallow: /img/ Disallow: /images/ Disallow: /logs/ Disallow: /ns-icons/ # Disallow: /F02/ Disallow: /S03/ Disallow: /bible/ Disallow: /electronics/ Disallow: /family/pics/ Disallow: /internet/ Disallow: /web/ # Disallow: /CS1/ # Disallow: /M02/ # Disallow: /cadresume/ # Disallow: /cis/ # Disallow: /csresume/ # Disallow: /mamoun/ # Disallow: /misresume/ # Disallow: /physics/ # Disallow: /physics122/ # Disallow: /pics/ # Disallow: /pres/ # Disallow: /resume/ # Disallow: /spaceshipclassroom/ # Disallow: /techresume/ # Disallow: /thumbs/ # Disallow: /webresume/