User-agent: * # disallow all files in these directories Disallow: /1/ Disallow: /stats/ Disallow: /*?* User-agent: Googlebot # disallow all files ending with these extensions Disallow: /1/ Disallow: /stats/ Disallow: /*?* Disallow: /*.js$ Disallow: /*.inc$ Disallow: /*.css$ Disallow: /*.gz$ Disallow: /*.cgi$ Disallow: /*.xhtml$ # allow google image bot to search all images User-agent: Googlebot-Image Disallow: /1/ Allow: /* # disallow adsense bot on entire site User-agent: Mediapartners-Google* Disallow: /* Allow: # allow AdWords bot on entire site User-agent: Adsbot-Google* Disallow: /1/ Allow: /* # disallow archiving site User-agent: ia_archiver Disallow: /1/ Allow: /* # disable duggmirror User-agent: duggmirror Disallow: /