# PropertyLake - sold prices, council tax bands, energy ratings and
# census data for every postcode and street in England and Wales.

# --- AI training crawlers: declined ---------------------------------------
# These collect training data. They send no visitor, leave no link and cite
# nothing. The pages here are an aggregation of public records, and the
# aggregation is the product. Enforced at the server as well as requested here.
User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# --- training opt-outs that exist only here --------------------------------
# Google-Extended and Applebot-Extended are not crawlers and have no user agent
# of their own - they are tokens that control whether content already fetched by
# Googlebot and Applebot may be used to train Gemini and Apple Intelligence.
# They can only be refused in this file, never at the server. Refusing them does
# not affect Google Search or Siri: those are Googlebot and Applebot, which are
# welcome below.
User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# --- AI search crawlers: welcome ------------------------------------------
# These fetch a page to answer someone's question and cite it back. That is a
# reader arriving, which is the same bargain a search engine offers.
User-agent: Claude-SearchBot
Disallow: /property/
Disallow: /api/
Allow: /

User-agent: OAI-SearchBot
Disallow: /property/
Disallow: /api/
Allow: /

# Fetched live because a person asked about this page, so also welcome.
User-agent: Claude-User
Disallow: /property/
Disallow: /api/
Allow: /

User-agent: ChatGPT-User
Disallow: /property/
Disallow: /api/
Allow: /

# --- everyone else --------------------------------------------------------
# /property/ is per-address and behind the paywall; there is nothing there for
# a crawler and 4.7M of them would be nothing but thin pages.
#
# /api/ is the page's own JSON. Googlebot was spending 51% of its crawl budget
# there - 32,318 requests to /api/geometry alone - because it renders the page
# and the map fetches its footprints. Nothing under /api/ is indexable and
# every figure is already in the server-rendered HTML, so the only thing lost
# is the map. That is decoration on a page made of text.
User-agent: *
Disallow: /property/
Disallow: /api/
Allow: /

# --- terms ----------------------------------------------------------------
# Reuse is welcome, including in an AI answer, with attribution to PropertyLake
# and a link to the page used. Full terms and a machine-readable summary:
#   https://propertylake.co.uk/licence
#   https://propertylake.co.uk/llms.txt
# Stated, not enforced: no directive here can compel attribution, and this is
# the ask rather than a control.

Sitemap: https://propertylake.co.uk/sitemap.xml