# Pets In The City - robots.txt
# https://www.petsinthecity.co.nz

User-agent: *
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
# Legacy HubSpot CTA/tracking paths — 301 to canonical pages; no crawl value
Disallow: /cs/
Disallow: /hs/cta/

# Sitemaps
Sitemap: https://www.petsinthecity.co.nz/sitemap.xml
Sitemap: https://www.petsinthecity.co.nz/sitemap-dog-names.xml

# Crawl-delay for polite crawling
Crawl-delay: 1

# Google specific
User-agent: Googlebot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /cs/
Disallow: /hs/cta/

# Bing specific
User-agent: Bingbot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /cs/
Disallow: /hs/cta/
Crawl-delay: 2

# =====================================================
# AI / LLM CRAWLERS
# We explicitly welcome AI assistants and search-style
# LLMs to crawl public content so PITC is represented
# in answers about Auckland & NZ pet care.
# See also: https://www.petsinthecity.co.nz/llms.txt
# =====================================================

# OpenAI - ChatGPT training crawler
User-agent: GPTBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# OpenAI - ChatGPT live browsing (when users ask questions)
User-agent: ChatGPT-User
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# OpenAI - SearchGPT / ChatGPT Search index
User-agent: OAI-SearchBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Anthropic - Claude training crawler
User-agent: anthropic-ai
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Anthropic - Claude live browsing
User-agent: Claude-Web
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Anthropic - new official user agent (2024+)
User-agent: ClaudeBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Google - Gemini / Bard training & AI Overviews
User-agent: Google-Extended
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

User-agent: Applebot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Meta AI (Llama, Meta AI assistant)
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

User-agent: FacebookBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Amazon - Alexa / Rufus
User-agent: Amazonbot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Microsoft Copilot (uses Bingbot above + this)
User-agent: CopilotBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# DuckDuckGo AI Assist
User-agent: DuckAssistBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Cohere
User-agent: cohere-ai
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

User-agent: cohere-training-data-crawler
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Mistral AI
User-agent: MistralAI-User
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# You.com
User-agent: YouBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Common Crawl - foundational dataset for many LLMs
User-agent: CCBot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Diffbot - structured data extraction used by AI tools
User-agent: Diffbot
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# ByteDance / TikTok - Doubao AI
User-agent: Bytespider
Allow: /
Disallow: /portal/
Disallow: /admin/
Disallow: /api/

# Host
Host: https://www.petsinthecity.co.nz