# Website Sitemap
Sitemap: https://www.deleukstetaartenshop.nl/sitemap.xml

# Crawlers Setup
User-agent: *

# Paths used by Magento's sorting or layered navigation
Disallow: /*/q/*_ASC*
Disallow: /*/q/*_DESC*
Disallow: /*limit=*
Disallow: /*order=*
Disallow: /*dir=*
Disallow: /*price=*
Disallow: /*multiplie=*
Disallow: /*product_list_order=*
Disallow: /*osCsid*
Disallow: /*cPath*
Disallow: /*?page=NaN
Disallow: /*?p=NaN

# ----------------------------------------------------
# Block bad crawlers
# ----------------------------------------------------
# third party search engine bot
User-agent: MJ12bot
Disallow: /

# Web-crawler by the company 'WiseGuys' to gather unstructured webdata
User-agent: Vagabondo
Disallow: /

# Chinese Search Engine
User-agent: BaiDuSpider
Disallow: /

# French Search Engine
User-agent: Exabot
Disallow: /

# Russian Search Engine
User-agent: Yandex
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: CFNetwork
Disallow: /

User-agent: TwengaBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: FatBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: seoscanners.net
Disallow: /

User-agent: SpeedySpider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /