# This says to apply these settings to ALL search engine spiders/crawlers
User-agent: *

# These settings will keep spiders from indexing your unwanted pages 
Disallow: /cpanel
Disallow: /contents
Disallow: /checkout
Disallow: /lib
Disallow: /theme
Disallow: /stats
Disallow: /auth
Disallow: /scheduler
Disallow: /video-processor
Disallow: /video-player