User-agent: *
Disallow: /dataset?*
Disallow: /dataset/activity/*
Disallow: /dataset/groups/*
Disallow: /dataset/showcases/*
Disallow: /dataset/*/issues/*
Disallow: /dataset/*/resource/*
Disallow: /datastore/*
Disallow: /datarequest/*
Disallow: /group/*?*
Disallow: /organization/*?*
Disallow: /showcase?*
Disallow: /issues/
Disallow: /revision/
Disallow: /user/*
Disallow: /api/
Disallow: /cgi-bin
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /*.php$
Disallow: /*.inc$
Disallow: /*.gz$
Disallow: /*.wmv$
Disallow: /*.cgi$
Disallow: /*.xhtml$
Disallow: /ar/
Disallow: /bg/
Disallow: /ca
Disallow: /cs_CZ/
Disallow: /da_DK/
Disallow: /de/
Disallow: /dv/
Disallow: /el/
Disallow: /en/
Disallow: /en_AU/
Disallow: /en_GB/
Disallow: /es/
Disallow: /es_AR/
Disallow: /fa_IR/
Disallow: /fi/
Disallow: /fr/
Disallow: /he/
Disallow: /hr/
Disallow: /hu/
Disallow: /id/
Disallow: /is/
Disallow: /it/
Disallow: /ja/
Disallow: /km/
Disallow: /ko_KR/
Disallow: /lt/
Disallow: /lv/
Disallow: /mn_MN/
Disallow: /my_MM/
Disallow: /ne/
Disallow: /nl/
Disallow: /no/
Disallow: /pl/
Disallow: /pt_BR/
Disallow: /pt_PT/
Disallow: /ro/
Disallow: /ru/
Disallow: /sk/
Disallow: /sl/
Disallow: /sq/
Disallow: /sr/
Disallow: /sr_Latn/
Disallow: /sv/
Disallow: /th/
Disallow: /tl/
Disallow: /tr/
Disallow: /uk_UA/
Disallow: /vi/
Disallow: /zh_CN/
Disallow: /zh_HK/
Disallow: /zh_TW/

User-agent: *
Crawl-delay: 15

User-agent: Jigsaw
Allow: /
User-agent: LinkCheck
Allow: /
User-agent: SiteCheck-sitecrawl
Allow: /
User-agent: Siteimprove_W3C_Validator
Allow: /

User-agent: AhrefsBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: BUbiNG
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Jorgee
Disallow: /
User-agent: larbin
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: netEstate NE Crawler
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: spbot
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: ZyBORG
Disallow: /