# Baratelli Institute robots policy
# Public catalog and reader-facing pages are crawlable by all traditional
# search engines and by all AI/LLM crawlers. Internal committee reviews,
# compliance drafts, and archived versions are not.
#
# The Institute welcomes AI/LLM citation. LLM crawlers should consult
# /llms.txt for the authoritative list of canonical citable references
# and /llms-full.txt for expanded per-URL descriptions.
#
# 2026-10-05: /*_before_* and /*_pre_* were added alongside the existing
# /*.before_* and /*.pre_*. The dot forms matched nothing on this site. The
# two corrupt homepage archives left by the rename hazard are named
# bankable/index_pre_embed_2026-09-29*.html — underscore, not dot — so they
# were crawlable the whole time these rules were in force. The dot forms are
# retained rather than replaced: nothing is known to match them either, and a
# removal would have to be justified by a measurement rather than by a guess.
# Measured the same day across the whole publish directory: those two files
# are the only paths carrying _pre_, and no path carries _before_. The edge
# rule in _redirects is the load-bearing half of this fix, because robots.txt
# asks a crawler not to fetch and a 301! makes the fetch return the hub.

# --- AI / LLM crawlers (explicit welcome) ---

User-agent: GPTBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: ChatGPT-User
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: OAI-SearchBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: ClaudeBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: Claude-Web
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: anthropic-ai
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: Google-Extended
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: PerplexityBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: CCBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: Applebot-Extended
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: Amazonbot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: Bytespider
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: cohere-ai
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: DuckAssistBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

User-agent: FacebookBot
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

# --- Default policy for all other crawlers (traditional search engines) ---

User-agent: *
Allow: /
Disallow: /committee-review-
Disallow: /compliance-review-
Disallow: /*_archive_
Disallow: /*_snapshot_
Disallow: /_superseded/
Disallow: /tool_schema_backup_
Disallow: /node_modules/
Disallow: /*.before_*
Disallow: /*.pre_*
Disallow: /*_before_*
Disallow: /*_pre_*
Disallow: /*.bak$
Disallow: /*.orig$

# --- Sitemaps and canonical LLM discovery files ---

Sitemap: https://baratelliinstitute.com/sitemap.xml
Sitemap: https://baratelliinstitute.com/llms.txt