# robots.txt for WordPress

User-agent: *
Disallow: /_derived/	# Frontpage, ancient
Disallow: /_themes/		# Frontpage, ancient
Disallow: /_private/	# Frontpage, ancient
Disallow: /htm/
Disallow: /lists/
Disallow: /sitespecific/
Disallow: /images/

Disallow: /ChecklistForSellingInformationProducts.html_files/
Disallow: /3290703go-vci/*
Disallow: /audio/
Disallow: /cgi-bin/*
Disallow: /cgi-bin/apf4/*
Disallow: /cgi-bin/clickbankbuddy/*
Disallow: /calendar/
Disallow: /crnano/
Disallow: /data/
Disallow: /ddtabmenufiles/
Disallow: /feedforall/
Disallow: /newsfeeds/
Disallow: /utils/
Disallow: /shared/*
Disallow: /shared/youtubevideosite/*
Disallow: /socialbookmarkscript/*
Disallow: /styles/*
Disallow: /youtubevideosite/*

# WordPress
Disallow: /feed/*
Disallow: /trackback/*
Disallow: /wp-admin/*
Disallow: /wp-content/*
Disallow: /wp-includes/*
Disallow: /xmlrpc.php
Disallow: /wp-
Allow: /wp-content/uploads/*
Disallow: /category/*/*
Disallow: /comments
Disallow: */comments
Disallow: */feed
Disallow: /manager
Disallow: */trackback

# from http://perishablepress.com/wordpress-robots-rules/
Disallow: ?wptheme=
Disallow: /*/wp-*
Disallow: /*/feed/*
Disallow: /*/*?s=*
Disallow: /*/*.js$
Disallow: /*/*.inc$
Disallow: /*/cgi-bin/*
Disallow: /*/blackhole/*
Disallow: /*/trackback/*
Disallow: /*/xmlrpc.php
Allow: /*/20*/wp-*
# Don't use Allow: /*  for a specific User-agent
# That over-rides the folders for User-Agent: * , then search engine has hundreds of youtubevideosite and apf4 links that can't follow
# as if are links on your site! (No, those are poorly defined links in comments on YouTube, etc)
# See test page https://www.google.com/webmasters/tools/ then under Crawl click on Blocked URLs

# Google ignores this, uses setting in Google Webmaster Central
Crawl-delay: 30

##### Must have an allow or disallow after Google lines, or Google obeys the next one it finds #####
User-agent: Googlebot
Disallow: /cgi-bin/
Disallow: /cgi-bin/apf4/
Disallow: /crnano/
Disallow: /wp-admin/*
Disallow: /wp-content/plugins/*
Disallow: /wp-content/themes/*
Disallow: /shared/*
Disallow: /shared/youtubevideosite/*
Allow: /googlefolder/

# Google Image
User-agent: Googlebot-Image
Disallow: /wp-admin/*
Disallow: /wp-content/plugins/*
Disallow: /wp-content/themes/*
Disallow: /shared/*
Disallow: /shared/youtubevideosite/*
Disallow: /cgi-bin/apf4/*
Allow: /googlefolder/

# Google AdSense
User-agent: Mediapartners-Google*
Disallow: /wp-admin/*
Disallow: /wp-content/plugins/*
Disallow: /wp-content/themes/*
Disallow: /shared/*
Disallow: /shared/youtubevideosite/*
Allow: /googlefolder/

# digg mirror
# User-agent: duggmirror
# Disallow: /

User-agent: *
Disallow: /shared/blackhole/