User-agent: *
Disallow: /wp-admin/
Disallow: /chat-api/
Allow: /wp-admin/admin-ajax.php
# 31-Aug-2026: /number/<n>/ai-report views (noindex, unlinked, zero indexed)
# and their REST endpoint were drawing ~271k Googlebot + ~39k GoogleOther +
# ~104k REST fetches/day mid-recovery. Blocked surgically — only the
# ai-report URL space, NOT /wp-json/ generally. Prefix match also covers the
# trailing-slash variant (it 301s to the bare form).
Disallow: /number/*/ai-report
Disallow: /wp-json/ct/v1/ai-report
# 23-Sep-2026: /check-uk-phone-number?n=<digits> deep links (both slash forms).
# Noindexed since 31-Aug and canonical to the bare checker URL, yet Googlebot
# was still fetching ~78k/day of them — pure rechecks of URLs that will never
# be indexed (23 URLs / 133 impressions in the last 14 days). Blocked so they
# leave the crawl queue. The checker page itself, its REST endpoint and the
# #n= fragment deep links are unaffected (fragments are never requested).
Disallow: /check-uk-phone-number?n=
Disallow: /check-uk-phone-number/?n=

# ---------------------------------------------------------------------------
# AI crawlers — all still fully allowed. Training bots are rate-limited.
#
# 31-Jul-2026: at midnight, with negligible human traffic, this server was
# taking 127,000 requests/hour and sitting at load 15 on 16 cores. 73% was
# bots, and AI crawlers alone were ~50% of all traffic — ClaudeBot on its own
# was 36,218 requests/hour (10 req/sec, 30.8% of everything).
#
# They were spending it on /number/ pages that serve noindex: of 400 sampled
# ClaudeBot URLs, 400 were noindex. GPTBot and Amazonbot likewise 100%.
# Every syntactically-valid UK number renders a full 136 KB page, so the
# crawlable space is effectively unbounded and no cache can help — measured
# repeat rate was 1.00-1.07 requests per distinct URL, i.e. a pure miss stream.
#
# Crawl-delay is applied only to the bots that collect TRAINING data, because
# throttling those costs us nothing we care about. The search/retrieval bots
# below are what actually produce AI citations, so they stay unthrottled.
# Googlebot and bingbot are deliberately untouched — they earn the revenue,
# and Google ignores Crawl-delay in any case.
#
# Anthropic, OpenAI, Amazon and Baidu all document support for Crawl-delay.
# Crawl-delay: 2 caps a bot at 0.5 req/sec — still 43,200 pages/day, far more
# than the ~8,470 number pages that are actually indexable.
# ---------------------------------------------------------------------------

# --- Training-data crawlers: throttled ---

User-agent: ClaudeBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: Amazonbot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: Bytespider
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: Baiduspider
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: Diffbot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: cohere-ai
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

User-agent: YouBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/
Crawl-delay: 2

# --- Search / retrieval crawlers: unthrottled, these drive AI citations ---
# Anthropic split ClaudeBot into three agents (Feb 2026). Claude-SearchBot
# indexes for search results and Claude-User fetches on a user's behalf, so
# both affect whether we get cited and neither is throttled here.

User-agent: Claude-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/

User-agent: Claude-User
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/

User-agent: ChatGPT-User
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/

User-agent: PerplexityBot
Allow: /
Disallow: /wp-admin/
Disallow: /chat-api/

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# AI / LLM data discovery files
# /llms.txt        — topical index of every page (llmstxt.org)
# /llms-full.txt   — full text corpus of cornerstone & pillar content
# /.well-known/ai.txt — AI usage policy (allows training, summarising, citing)

Sitemap: https://connection-technologies.co.uk/sitemap_index.xml
Sitemap: https://connection-technologies.co.uk/image-sitemap.xml
Sitemap: https://connection-technologies.co.uk/news-sitemap.xml
Sitemap: https://connection-technologies.co.uk/video-sitemap.xml