# ============================================================
# robots.txt for taiuru.co.nz
# Taiuru & Associates Ltd: Dr Karaitiana Taiuru
# Māori AI, Māori Data Sovereignty & Tech Governance
# ============================================================
#
# A crawler follows ONLY the most specific group that names it.
# Every group below therefore repeats the same Disallow rules,
# so that no named crawler is let into admin or utility paths.
#
# The AI crawlers are split into two groups:
#   Group 1: search, citation and live retrieval (answers that
#            link back to this site)
#   Group 2: model training (content used to train AI models)
# To stop AI training while staying citable in AI search,
# change "Allow: /" to "Disallow: /" in Group 2 only.
# ============================================================


# ------------------------------------------------------------
# DEFAULT: all crawlers, including Googlebot and Bingbot
# Googlebot controls Google Search and AI Overviews.
# Bingbot controls Bing and Microsoft Copilot.
# ------------------------------------------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?rest_route=
Disallow: /?s=
Disallow: /search/


# ------------------------------------------------------------
# GROUP 1: AI SEARCH, CITATION AND LIVE RETRIEVAL
# These fetch pages to answer a person's question and cite
# the source.
# ------------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: meta-externalfetcher
User-agent: YouBot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?rest_route=
Disallow: /?s=
Disallow: /search/


# ------------------------------------------------------------
# GROUP 2: AI MODEL TRAINING
# These collect content that may be used to train AI models.
# Currently allowed. Change "Allow: /" to "Disallow: /" here
# to opt out of training; Group 1 citation is not affected.
# ------------------------------------------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: CCBot
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /?rest_route=
Disallow: /?s=
Disallow: /search/


# ------------------------------------------------------------
# SITEMAP
# The Yoast index lists every child sitemap.
# ------------------------------------------------------------
Sitemap: https://www.taiuru.co.nz/sitemap_index.xml

# LLM index (informational; not a robots.txt directive):
# https://www.taiuru.co.nz/llms.txt