# Ausking robots.txt — Toni 2026-10-26
# Domain: https://auskingvisa.com
#
# CHANGELOG:
#   2026-08-30 — FLIPPED all AI bots to Disallow (Carol: "block the ai")
#   2026-09-04 — Added facebookexternalhit allow for /img/ so social link previews work
#   2026-10-27 — blocked backup/preview trees + duplicate working copies from crawl
#                (sitemap rebuilt from live pages: 334 real URLs)
#   2026-10-26 — SEPARATED AI citation bots from AI training bots (Carol CI review).
#                Search/citation bots are now ALLOWED; training-only bots stay blocked.
#                Also unblocked /img/ + /images/ for all crawlers (Google Image Search +
#                AI Mode image results were being suppressed by the * group).
#                Reference:
#                  OpenAI    developers.openai.com/api/docs/bots  (OAI-SearchBot vs GPTBot)
#                  Anthropic Claude-SearchBot / Claude-User / ClaudeBot are independent controls
#                  Google    Googlebot is NOT blocked -> AI Overviews / AI Mode can still cite us
#                            (Google-Extended controls Gemini training/grounding -> stays blocked)

# ===== Social crawlers =====
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# ===== AI CITATION / SEARCH bots — ALLOWED =====
# These only discover and index content for search/citation surfaces.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ===== AI TRAINING bots — BLOCKED =====
# Training-only crawlers: content is not used to train foundation models.
User-agent: GPTBot
Disallow: /

User-agent: GPTBot-User
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# ===== All crawlers =====
# NOTE: /img/ and /images/ are intentionally ALLOWED here. Blocking them also blocked
# Googlebot from image assets, suppressing Google Image Search and AI Mode image results.
User-agent: *
Allow: /
Disallow: /vendor/
Disallow: /_backup_navtemplate_20261016_1742/
Disallow: /vn_backup_20261015/
Disallow: /_backup_seoaudit_20261022_0903/
Disallow: /si/
Disallow: /VN/
Disallow: /*preview
Disallow: /*BACKUP
Disallow: /*live_backup
Disallow: /*__sample
Disallow: /*__template
Disallow: /css/
Disallow: /js/
Disallow: /fonts/
Disallow: /data/
Disallow: /test/
Disallow: /cgi-bin/
Disallow: /Toni_seo/
Disallow: /.well-known/

# AI-readable site summary:
#   https://auskingvisa.com/llms.txt
# (a discovery aid for LLM crawlers — it is NOT what makes the site citable)

# Sitemaps
Sitemap: https://auskingvisa.com/sitemap.xml