# Realty of India — robots.txt
# ═══════════════════════════════════════════════════════════════════
# LAUNCH MODE: indexable.
# ═══════════════════════════════════════════════════════════════════
# AI policy: CITABLE, NOT TRAINABLE. ROI competes for citations in
# ChatGPT / Perplexity / Google AI Overviews / Gemini, so every SEARCH +
# CITATION crawler is allowed. The TRAINING-only crawlers are disallowed
# (training gives no attribution/traffic back; our listing data is the
# moat). The major engines run SEPARATE bots for search vs training, so
# opting out of training costs ZERO citation reach. The same preference is
# declared machine-readably via Content-Signal in the default group below.
# See /llms.txt for the curated map of pages we want LLMs to cite.
# (Pairs with FEATURE_PUBLIC_INDEXABLE=true, which stops the noindex meta.)

# ── Search + AI answer engines: DELIBERATELY NO GROUP OF THEIR OWN ─
# ⚠ DO NOT re-add `User-agent: Googlebot` / `Bingbot` / `PerplexityBot`
#   (etc.) groups here, however harmless an `Allow: /` looks.
#
# RFC 9309 §2.2.1, and Google's own spec in the same words: a crawler
# obeys the SINGLE most specific matching group and ignores every other
# one — "user agent specific groups and global groups (*) are not
# combined."
#
# This file used to open with 14 named groups each containing only
# `Allow: /`. Each of those crawlers therefore matched a group saying "you
# may fetch anything" and never read the `User-agent: *` block below — so
# all 52 rules in it, including every facet rule written in response to
# the 2026-08-09 crawl outage, applied to nobody who mattered.
#
# Measured on prod across the 11 retained days of nginx logs, BEFORE this
# fix (2026-08-20):
#     Googlebot requests ................. 14,691
#     ...carrying a query string .......... 3,715  (25.3%)
#     hits on /api/ads/slot (Disallowed) .. 1,530
#     hits on /auth/phone   (Disallowed) ..    90
#     ?roi= 249 · ?sort= 205 · ?bhk= 193 · ?rera= 181 · ?gated= 122
# The control, same file and same window: the TRAINING crawlers below DO
# have groups of their own, and obeyed them exactly —
#     GPTBot 4 · CCBot 1 · Bytespider 0
# Same file, opposite outcome. The only variable is which group matched.
#
# A crawler named in NO group falls through to `*` and inherits the full
# rule set, which is what these rules always meant. Search and AI-answer
# crawlers remain allowed everywhere — `*` opens with `Allow: /` — they
# simply honour the Disallows now too. The citation policy is unchanged
# and is enforced by the TRAINING-only groups below, which must KEEP their
# own groups precisely BECAUSE a named group overrides `*`.
#
# Pinned by scripts/lint-robots-groups.php (blocking CI gate).

# ── AI TRAINING crawlers (model-training only — DISALLOW) ──────────
# "Citable, not trainable": each engine's search/citation bot above is a
# SEPARATE agent that stays allowed, so blocking these costs no citations.
#   GPTBot            = OpenAI model training
#   CCBot             = Common-Crawl aggregator (bulk training corpus)
#   Google-Extended   = Gemini training opt-out (Googlebot + AI Overviews UNAFFECTED)
#   Applebot-Extended = Apple-Intelligence training opt-out (Applebot search UNAFFECTED)
#   Bytespider        = ByteDance training scraper (no citation surface)
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Bytespider
Disallow: /

# ── Default ────────────────────────────────────────────────────────
# Content-Signal (contentsignals.org) declares AI-usage preference
# machine-readably, mirroring the crawler rules above: allow search
# indexing + real-time answer-grounding (ai-input), opt out of model
# training (ai-train).
User-agent: *
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# ── Owner-private + transactional areas ────────────────────────────
# ⚠ /admin/, /auth/, /compose/, /onboarding, /claim, /intros/,
#   /conversations/, /saved and /preview/ are DELIBERATELY NOT LISTED
#   HERE. They are handled by `noindex` (App\Domain\Seo\RobotsPolicy),
#   and a Disallow would DEFEAT that noindex — you cannot have both.
#
# Disallow governs CRAWLING; noindex governs INDEXING; and a crawler that
# is not allowed to fetch a page can never READ the noindex on it. Google
# is explicit: "the page can still appear in search results… if other
# pages link to it. To properly prevent indexing, remove the crawl block."
# The result of doing both is the "Indexed, though blocked by robots.txt"
# state — the URL stays in the index permanently, with no snippet.
#
# That was live here. `Disallow: /auth/` sat in this file the whole time
# /auth/phone was indexed and serving as a brand-query SITELINK. It did
# nothing, for the group-precedence reason above — and the moment that was
# fixed on 2026-08-20, it would have started actively PRESERVING the very
# index entries the noindex shipped to remove. Both halves landed in
# c16fa77e and this block is the same-day correction.
#
# Rule: a path that emits `noindex` must stay crawlable until it drops out
# of the index. These pages cost almost nothing to crawl (a handful of
# URLs, and every authed one answers a crawler with a 302 to login).
# Pinned by scripts/lint-robots-groups.php, which now FAILS if any
# RobotsPolicy prefix reappears below.
#
# /api/ stays: it returns JSON, renders no Blade layout, and so cannot
# carry a meta tag. Disallow is the only tool that reaches it.
Disallow: /api/
# Listing edit / mark-sold / withdraw — owner actions, not public content.
Disallow: /listings/*/edit
# Filtered/sorted index URLs are duplicate-content noise.
Disallow: /matches?*
Disallow: /listings?*
Disallow: /buyers?*

# ── Faceted-filter crawl-budget killers (per Google guidance) ──
# `?*param=` matches the param at ANY position in the query string —
# the old `/*?param=` form only matched param-first URLs, so facet
# crossproducts like ?coliving_brand=x&pg_gender=y slipped every rule.
# On 2026-08-09 a crawl of exactly those permutations (52k IPs, every
# URL a CF cache MISS) took the origin down. Every param below
# canonicalizes to the clean path URL, so these rules cost ZERO
# indexing — they only stop compliant crawlers from walking the
# infinite filter crossproduct.
#
# ⚠ These became LIVE for Googlebot on 2026-08-20 (see the group note at
#   the top — they had been inert since they were written). Two knowingly
#   accepted consequences:
#     · `?page=` blocks paginated SRPs. Listing discovery does not depend
#       on them — every listing ships in sitemap-listings.xml.
#     · `?locality=` covers the WebSite SearchAction target
#       (/search?locality={term}). That URL is a 301 to the canonical path
#       SRP, never a destination, and the sitelinks search box it was
#       written for was removed by Google on 2024-11-21, so the markup has
#       no consumer. Deliberately NOT special-cased with an Allow.
Disallow: /*?*amenities=
Disallow: /*?*beach=
Disallow: /*?*bhk=
Disallow: /*?*budget_label=
Disallow: /*?*budget_max=
Disallow: /*?*budget_min=
Disallow: /*?*capacity_min=
Disallow: /*?*city=
Disallow: /*?*coliving_brand=
Disallow: /*?*commercial_subtype=
Disallow: /*?*facing=
Disallow: /*?*furnishing=
Disallow: /*?*gated=
Disallow: /*?*listing_kind=
Disallow: /*?*locality=
Disallow: /*?*oc=
Disallow: /*?*owner=
Disallow: /*?*page=
Disallow: /*?*pet_friendly=
Disallow: /*?*pg_gender=
Disallow: /*?*pool=
Disallow: /*?*possession=
Disallow: /*?*possession_within=
Disallow: /*?*property_type=
Disallow: /*?*ready_to_move=
Disallow: /*?*rera=
Disallow: /*?*roi=
Disallow: /*?*room_sharing=
Disallow: /*?*sort=
Disallow: /*?*stay_max=
Disallow: /*?*transaction_role=
Disallow: /*?*verified=
Disallow: /*?*view=
Disallow: /*?*with_photos=
Disallow: /*?*with_videos=
Disallow: /*?*zoning=

# ── Ad Spots click redirector ──────────────────────────────────────
# /ads/c/{creative} is a server-side counter+302 redirector — not a
# crawlable resource. Disallow prevents AI agents and SEO crawlers from
# following ad links (they'd pollute click counts AND get robots-baited
# into 404s once the booking expires).
Disallow: /ads/c/

# Sitemaps — sitemap.xml is a sitemap-index declaring the per-segment
# files; key segments are also listed individually for GSC pickup.
Sitemap: https://realtyofindia.com/sitemap.xml
Sitemap: https://realtyofindia.com/sitemap-cities.xml
Sitemap: https://realtyofindia.com/sitemap-localities.xml
Sitemap: https://realtyofindia.com/sitemap-listings.xml