# Yandex search, resource intensive
User-agent: YandexBot
Disallow: /

# Yahoo search, can be intensive
User-agent: Slurp
Disallow: /

# Exalead search
User-agent: ExaBot
Disallow: /

# Petal search, Huawei
User-agent: PetalBot
Disallow: /

# Anthropic, AI, aggressive
User-agent: ClaudeBot
Disallow: /

# Bytedance AI
User-agent: Bytespider
Disallow: /

# Alexa AI
User-agent: Amazonbot
Disallow: /

# Common Crawl, creates large datasets
User-agent: CCBot
Disallow: /

# Mozilla SEO bot
User-agent: DotBot
Disallow: /

# Extensive SEO crawling bot
User-agent: AhrefsBot
Disallow: /

# Aggressive SEO bot
User-agent: SemrushBot
Disallow: /

# SEO bot
User-agent: Sitebulb
Disallow: /

# Tencent
User-agent: Sogou
Disallow: /

# Distributed web crawler that generates lots of traffic
User-agent: MJ12bot
Disallow: /

User-agent: BaiduSpider
Disallow: /

User-agent: *
Crawl-delay: 5

Disallow: /content/download
Disallow: /mtmte
Disallow: /sok
# pcform is not unique pages, just a filter
Disallow: /*?pcform

Sitemap: https://www.nupi.no/sitemap.xml