# Sitemap Sitemap: https://www.loothoarder.com/sitemaps.xml # Allow AI retrieval bots that drive referral traffic User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: Claude-User Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Applebot Allow: / User-agent: DuckDuckBot Allow: / # Block AI training bots that scrape without sending traffic back User-agent: GPTBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: CCBot Disallow: / User-agent: meta-externalagent Disallow: / User-agent: FacebookBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: anthropic-ai Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Diffbot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: YouBot Disallow: / User-agent: cohere-ai Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: peer39_crawler Disallow: / User-agent: peer39_crawler/1.0 Disallow: / User-agent: AI2Bot Disallow: / User-agent: Timpibot Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: Scrapy Disallow: / # SEO scrapers (these clutter logs and offer no value) User-agent: SemrushBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SeznamBot Disallow: / User-agent: PetalBot Disallow: / User-agent: SleepBot Disallow: / # Universal rules for all bots User-agent: * # Block bot-favorite filter URL combinations Disallow: /*?filter_ Disallow: /*?query_type_ Disallow: /*&filter_ # Don't crawl admin, cart, checkout, account Disallow: /wp-admin/ Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ Disallow: /*?add-to-cart= Disallow: /*?remove_item= # Allow wp-admin/admin-ajax.php (needed for frontend features) Allow: /wp-admin/admin-ajax.php # Crawl delay for less aggressive scraping (Yandex/others respect this) Crawl-delay: 10