# Block Amazonbot - Amazonbot is a web crawler used by Amazon to index search results, allowing the Alexa AI Assistant to provide more accurate answers to user questions.
User-agent: Amazonbot
Disallow: /
# Block meta-externalagent (likely Facebook scraping bot) - Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly.
User-agent: meta-externalagent
Disallow: /
# Block ClaudeBot (Anthropic AI bot) - ClaudeBot is a web crawler used to download training data for training LLMs (Large Language Models).
User-agent: ClaudeBot
Disallow: /
# Block PetalBot - PetalBot is an automatic program of the Petal search engine.
User-agent: PetalBot
Disallow: /
# Block Amazonbot - Amazonbot is a web crawler used by Amazon to index search results, allowing the Alexa AI Assistant to provide more accurate answers to user questions.
User-agent: Amazonbot
Disallow: /
# Block meta-externalagent (likely Facebook scraping bot) - Meta-ExternalAgent is a web crawler used by Meta to download training data for its AI models and improve its products by indexing content directly.
User-agent: meta-externalagent
Disallow: /
# Block ClaudeBot (Anthropic AI bot) - ClaudeBot is a web crawler used to download training data for training LLMs (Large Language Models).
User-agent: ClaudeBot
Disallow: /
# Block PetalBot - PetalBot is an automatic program of the Petal search engine.
User-agent: PetalBot
Disallow: /
# Block AhrefsBot (SEO crawler) - AhrefsBot is a web crawler that powers the database for both Ahrefs, an online data toolset, and Yep, a revenue-sharing web search engine.
User-agent: AhrefsBot
Disallow: /
# Block DotBot (SEO and link analysis bot)
User-agent: DotBot
Disallow: /
# Block MJ12bot (Distributed crawler)
User-agent: MJ12bot
Disallow: /
# Block DataForSeoBot (SEO data crawler)
User-agent: DataForSeoBot
Disallow: /
# Block MegaIndex (SEO analytics bot)
User-agent: MegaIndex
Disallow: /
# Block BLEXBot (SEO crawler)
User-agent: BLEXBot
Disallow: /
# Block CCBot (SEO and analytics bot)
User-agent: CCBot
Disallow: /
# Block Seekport Crawler (General purpose crawler)
User-agent: Seekport Crawler
Disallow: /
# Block Screaming Frog SEO Spider (Website analysis tool)
User-agent: Screaming Frog SEO Spider
Disallow: /
# Block SeznamBot (Czech search engine bot)
User-agent: SeznamBot
Disallow: /
# Block OnCrawl (SEO and website analysis bot)
User-agent: OnCrawl
Disallow: /
# Block Serpstatbot (Advanced backlink tracking bot)
User-agent: serpstatbot
Disallow: /
# Block SeekportBot (Search engine bot)
User-agent: SeekportBot
Disallow: /
# Block GeedoProductSearch (Product search crawler)
User-agent: GeedoProductSearch
Disallow: /
# Block YandexBot (Russian search engine bot)
User-agent: YandexBot
Disallow: /
# Block Go-http-client (Generic HTTP client, often used by bots)
User-agent: Go-http-client
Disallow: /
# Block GPTBot (OpenAI crawler for GPT models)
User-agent: GPTBot
Disallow: /
# Block bingbot (Microsoft Bing search engine bot)
User-agent: bingbot
Disallow: /
Sitemap: https://www.petitegeneve.com/sitemap.xml