# Define access-restrictions for robots/spiders
# http://www.robotstxt.org/wc/norobots.html

User-agent: TweetmemeBot
Disallow: /
User-agent: yoozBot
Disallow: /
User-agent: JamesBOT
Disallow: /
User-agent: yacybot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: 360Spider
Disallow: /
User-agent: Screaming Frog SEO Spider
Disallow: /
User-agent: SEMrushBot
Disallow: /
User-agent: uCrawler
Disallow: /
User-agent: Itslearning
Disallow: /
User-agent: Scanmine
Disallow: /
User-agent: Pywikibot
Disallow: /
User-agent: wlc Pywikibot
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: Cliqzbot
Disallow: /
User-agent: linkdexbot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: gocrawl
Disallow: /
User-agent: trendictionbot
Disallow: /
User-agent: ltx71
Disallow: /
User-agent: GarlikCrawler 
Disallow: /
User-agent: AhrefsBot 
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: Mediatoolkitbot
Disallow: /
User-agent: DeuSu
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: BUbiNG
Disallow: /
User-agent: spbot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: Yeti
Disallow: /
User-agent: dotbot
Disallow: /
User-Agent: MJ12bot
Disallow: /
User-Agent: Exabot
Disallow: /
User-Agent: BLEXBot
Disallow: /
User-Agent: SEMrushBot
Disallow: /
User-Agent: coccocbot
Disallow: /
User-Agent: proximic
Disallow: /
User-Agent: AwarioRssBot
Disallow: /
User-Agent: YaK
Disallow: /
User-Agent: Yandex
Disallow: /
User-Agent: Go-http-client
Disallow: /
User-Agent: Genieo
Disallow: /
User-Agent: Mail.RU_Bot
Disallow: /
User-Agent: MojeekBot
Disallow: /
User-Agent: Qwantify
Disallow: /
User-Agent: changedetection
Disallow: /
User-Agent: coccocbot-image
Disallow: /
User-Agent: coccocbot-web
Disallow: /
User-Agent: proximic
Disallow: /
User-Agent: spbot
Disallow: /
User-Agent: Riddler
Disallow: /
User-Agent: Sogou
Disallow: /
User-Agent: Sogou web spider
Disallow: /
User-Agent: istellabot
Disallow: /

# Exclude stuff that uses much CPU
User-agent: *
Disallow: /biografier/historisk/minnedager/list.html
Disallow: /biografier/historisk/minnedager/past.html
Disallow: /biografier/historisk/minnedager/day.html
Disallow: /biografier/historisk/minnedager/date.html
Disallow: /biografier/historisk/minnedager/week.html
Disallow: /biografier/historisk/minnedager/month.html

# By default we allow robots to access all areas of our site 
# already accessible to anonymous users

User-agent: *
Disallow: /biografier/innenriks/aeinarsson
Disallow: /biografier/innenriks/gsandanger

# Add Googlebot-specific syntax extension to exclude forms 
# that are repeated for each piece of content in the site 
# the wildcard is only supported by Googlebot
# http://www.google.com/support/webmasters/bin/answer.py?answer=40367&ctx=sibling

User-Agent: *
Crawl-delay: 90
Disallow: /services/
Disallow: /++*
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*ics_view$
Disallow: /*vcs_view$
Disallow: @@
Disallow: /@@search
Disallow: /search
Disallow: /*?*
Disallow: /*=*
Disallow: /biografier/historisk/minnedager