# /robots.txt file # User-agent: Slurp Crawl-delay: 20 Disallow: /~ Disallow: /. Disallow: /cgi-bin/ Disallow: /scripts/ Disallow: /Scripts/ Disallow: /icons/ Disallow: /Icons/ Disallow: /images/ Disallow: /Images/ Disallow: /archives_restricted/ Disallow: /archives_private/ Disallow: /gallery_restricted/ Disallow: /gallery_private/ Disallow: /library_restricted/ Disallow: /library_private/ Disallow: /other_restricted/ Disallow: /other_private/ User-agent: * Disallow: /~ Disallow: /. Disallow: /cgi-bin/ Disallow: /scripts/ Disallow: /Scripts/ Disallow: /icons/ Disallow: /Icons/ Disallow: /images/ Disallow: /Images/ Disallow: /archives_restricted/ Disallow: /archives_private/ Disallow: /gallery_restricted/ Disallow: /gallery_private/ Disallow: /library_restricted/ Disallow: /library_private/ Disallow: /other_restricted/ Disallow: /other_private/