Sitemap: https://www.opensanctions.org/sitemap.xml
Sitemap: https://www.opensanctions.org/entities/sitemap.xml

# Search engines (classic + AI-search that links back, + live user-fetch),
# plus social link-preview/unfurl bots so shared entity links render a card.
# Full access EXCEPT private/app paths and the bulk-data dump endpoints.
# Entity and search pages stay crawlable here so organic discovery keeps working.
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: DuckAssistBot
User-agent: DuckDuckGo-Favicons-Bot
User-agent: Applebot
User-agent: PerplexityBot
User-agent: OAI-SearchBot
User-agent: Claude-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: facebookexternalhit
User-agent: facebookcatalog
Disallow: /account
Disallow: /admin
Disallow: /auth
Disallow: /invalidate
Disallow: /issues
Disallow: /statements
Disallow: /advancedsearch
Disallow: /impressum

# SEO/backlink-intelligence crawlers and aggressive scrapers: no value, heavy load.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DataForSeoBot
User-agent: dotbot
User-agent: BLEXBot
User-agent: serpstatbot
User-agent: Barkrowler
User-agent: Bytespider
User-agent: Meta-ExternalAgent
User-agent: PetalBot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgili
User-agent: Webzio
User-agent: Timpibot
User-agent: magpie-crawler
Disallow: /

# Everyone else: AI-training crawlers (GPTBot, ClaudeBot, CCBot, Amazonbot,
# Google-Extended, ...) and any unrecognised bot.
# Marketing / docs / catalog pages are open; bulk data and private paths are not.
# Crawl-delay throttles these to reduce server load (honored by many, ignored by some).
User-agent: *
Crawl-delay: 10
Disallow: /entities
Disallow: /search
Disallow: /statements
Disallow: /advancedsearch
Disallow: /issues
Disallow: /account
Disallow: /admin
Disallow: /auth
Disallow: /invalidate
Disallow: /impressum