robots.txt

User-agent: * 
Disallow: /cgi-bin/
Disallow: /core/
Disallow: /classes/
Disallow: /config/
Disallow: /content/
Disallow: /error404/
Disallow: /log/
Disallow: /template/
Disallow: /cache/