User-agent: e-SocietyRobot Disallow: / # Bad bot - Often ignores robots.txt - Waste of bandwidth User-agent: Twiceler Disallow: / # W3C Bot which checks all links on the page - consumes bandwidth User-Agent: W3C-checklink Disallow: / # Stop following PHPSESSIDs User-Agent: MJ12bot Disallow: /forum/index.php?PHPSESSID # Google - Most Important bot # Supports * and $ User-agent: Googlebot Disallow: /forum/*;wap Disallow: /forum/*;wap2 Disallow: /forum/*;imode Disallow: /forum/*sa=showPosts Disallow: /forum/*prev_next # Also need the default stuff aswell # Default SMF Folders Disallow: /forum/attachments/ Disallow: /forum/Packages/ Disallow: /forum/Smileys/ Disallow: /forum/Sources/ Disallow: /forum/Themes/ # Default SMF Actions Disallow: /forum/index.php?action=activate Disallow: /forum/index.php?action=admin Disallow: /forum/index.php?action=emailuser Disallow: /forum/index.php?action=findmember Disallow: /forum/index.php?action=help Disallow: /forum/index.php?action=helpadmin Disallow: /forum/index.php?action=login Disallow: /forum/index.php?action=logout Disallow: /forum/index.php?action=mlist Disallow: /forum/index.php?action=modifykarma Disallow: /forum/index.php?action=pm Disallow: /forum/index.php?action=post Disallow: /forum/index.php?action=printpage Disallow: /forum/index.php?action=profile Disallow: /forum/index.php?action=recent Disallow: /forum/index.php?action=register Disallow: /forum/index.php?action=reminder Disallow: /forum/index.php?action=search Disallow: /forum/index.php?action=theme Disallow: /forum/index.php?action=unread Disallow: /forum/index.php?action=unreadreplies Disallow: /forum/index.php?action=verificationcode Disallow: /forum/index.php?action=who Disallow: /forum/index.php?theme= Disallow: /forum/index.php?jumpto= Disallow: /forum/index.php?action=sendtopic # Yahoo - Too aggressive # So limit it as much as possible. User-agent: Slurp # Disallow Everything Disallow: / # Now allow bits and then disallow bits Allow: /forum/sitemap.xml$ Allow: /forum/robots.txt$ Allow: /forum/index.php$ Allow: /forum/index.php?topic=*.0$ Allow: /forum/index.php?topic=*.*0$ Allow: /forum/index.php?topic=*.*5$ Allow: /forum/index.php?board=*.0$ Allow: /forum/index.php?board=*.*0$ Allow: /forum/index.php?board=*.*5$ # But don't allow these Disallow: /forum/index.php?*.msg Disallow: /forum/index.php?topic=*.msg*0$ Disallow: /forum/index.php?topic=*.msg*5$ Disallow: /forum/index.php?*.new # Anything with a ; disallow Disallow: /forum/index.php?*;* # Arcade Related Allow: /forum/index.php?action=arcade$ Allow: /forum/index.php?action=stats$ Allow: /forum/index.php?action=arcade;sa=play;game= # Catch all (remainder) # Will be followed by any bots other than ones identified above # Uses BASIC robots.txt directives without wildcards, end-anchors etc # So Spiders should understand these (including MSNBOT) User-agent: * # Default SMF Folders Disallow: /forum/attachments/ Disallow: /forum/Packages/ Disallow: /forum/Smileys/ Disallow: /forum/Sources/ Disallow: /forum/Themes/ # Default SMF Actions Disallow: /forum/index.php?action=activate Disallow: /forum/index.php?action=admin Disallow: /forum/index.php?action=emailuser Disallow: /forum/index.php?action=findmember Disallow: /forum/index.php?action=help Disallow: /forum/index.php?action=helpadmin Disallow: /forum/index.php?action=login Disallow: /forum/index.php?action=logout Disallow: /forum/index.php?action=mlist Disallow: /forum/index.php?action=modifykarma Disallow: /forum/index.php?action=pm Disallow: /forum/index.php?action=post Disallow: /forum/index.php?action=printpage Disallow: /forum/index.php?action=profile Disallow: /forum/index.php?action=recent Disallow: /forum/index.php?action=register Disallow: /forum/index.php?action=reminder Disallow: /forum/index.php?action=search Disallow: /forum/index.php?action=theme Disallow: /forum/index.php?action=unread Disallow: /forum/index.php?action=unreadreplies Disallow: /forum/index.php?action=verificationcode Disallow: /forum/index.php?action=who Disallow: /forum/index.php?theme= Disallow: /forum/index.php?jumpto= Disallow: /forum/index.php?action=sendtopic