#                                                                             
#          ██████  ██    ██  █████  ███    ██ ████████  █████               
#         ██    ██ ██    ██ ██   ██ ████   ██    ██    ██   ██              
#         ██    ██ ██    ██ ███████ ██ ██  ██    ██    ███████              
#         ██ ▄▄ ██ ██    ██ ██   ██ ██  ██ ██    ██    ██   ██              
#          ██████   ██████  ██   ██ ██   ████    ██    ██   ██              
#             ▀▀                                                           
#                                                                             
#          ██████  ██  ██████  ██ ████████  █████  ██                        
#          ██   ██ ██ ██       ██    ██    ██   ██ ██                        
#          ██   ██ ██ ██   ███ ██    ██    ███████ ██                        
#          ██   ██ ██ ██    ██ ██    ██    ██   ██ ██                        
#          ██████  ██  ██████  ██    ██    ██   ██ ███████                   
#                                                                             

Sitemap: https://quantadigital.co.uk/sitemap.xml

Content-Signal: ai-train=yes, search=yes, ai-input=yes

# allowed common search engines
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: MSNBot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Bravebot
User-agent: Ecosia
User-agent: Applebot
User-agent: SeznamBot
User-agent: Qwantbot
User-agent: MojeekBot
Allow: /

# disallow non targeted
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Sogou web spider
User-agent: Sogou spider2
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: SemrushBot-SWA
User-agent: PetalBot
User-agent: EtaoSpider
Disallow: /

# live search AIs (all allowed)
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: GoogleAgent-Mariner
User-agent: GoogleAgent-Shopping
User-agent: Gemini-Deep-Research
User-agent: meta-externalfetcher
User-agent: meta-webindexer
User-agent: Grok-DeepSearch
User-agent: MistralAI-User
User-agent: CopilotNative
User-agent: CopilotSapphire
User-agent: Manus-User
User-agent: YouBot
User-agent: iaskspider
User-agent: DuckAssistBot
User-agent: iAskBot
User-agent: PhindBot
User-agent: Andibot
Allow: /

# commom learning AIs (all allowed)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Google-NotebookLM
User-agent: Google-CloudVertexBot
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: xAI-Grok
User-agent: GrokBot
User-agent: Copilot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: AI2Bot-DeepResearchEval
User-agent: laion-huggingface-processor
User-agent: Crawl4AI
User-agent: Devin
Allow: /

# non-common learning AIs
User-agent: Bytespider
User-agent: DeepSeekBot
User-agent: ChatGLM-Spider
User-agent: PanguBot
User-agent: SBIntuitionsBot
User-agent: WRTNBot
User-agent: Scrapy
User-agent: img2dataset
User-agent: FriendlyCrawler
User-agent: VelenPublicWebCrawler
User-agent: Brightbot
User-agent: ISSCyberRiskCrawler
User-agent: imgproxy
User-agent: AddSearchBot
User-agent: Anomura
User-agent: BuddyBot
User-agent: Channel3Bot
User-agent: Cloudflare-AutoRAG
User-agent: CloudflareBrowserRenderingCrawler
User-agent: Cotoyogi
User-agent: Datenbank Crawler
User-agent: Echobot Bot
User-agent: EchoboxBot
User-agent: factset_spyderbot
User-agent: Google-Firebase
User-agent: GoogleOther
User-agent: IbouBot
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: imageSpider
User-agent: Kangaroo Bot
User-agent: KlaviyoAIBot
User-agent: KunatoCrawler
User-agent: Linguee Bot
User-agent: LinkupBot
User-agent: netEstate Imprint Crawler
User-agent: Panscient
User-agent: panscient.com
User-agent: Poggio-Citations
User-agent: QualifiedBot
User-agent: ShapBot
User-agent: Sidetrade indexer bot
User-agent: TavilyBot
User-agent: TerraCotta
User-agent: TwinAgent
User-agent: webzio-extended
User-agent: YaK
User-agent: ZanistaBot
User-agent: omgili
User-agent: wpbot
User-agent: aiHitBot
User-agent: Thinkbot
Disallow: /

User-agent: *
Crawl-delay: 1
Disallow: /admin/
Disallow: /subscribe/
Disallow: /unsubscribe/
Disallow: /login
Disallow: /two-factor