User-agent: *
Allow: /
Disallow: /company-data/

# Every page here is meant to be readable by people and by machines. Nothing is
# gated, nothing is cloaked, and no crawler is served different content.
# /company-data/ holds the raw JSON and SEC snapshots behind each company page. They
# stay public and linked for people, the API and the MCP servers. General search
# crawlers skip them because every one is served noindex and the pages that cite
# them are what a search should find; skipping them keeps crawl time on those pages.

# Answer engines, and the fetchers that open a page when a person asks an assistant
# to, read everything, the raw company data included, so an answer can quote a
# filing's exact figure from its source. Each has its own group because a crawler
# obeys only the most specific group that names it.
User-agent: Bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Search-marketing crawlers (backlink and keyword tools) are kept off the company
# catalogue, about 900,000 pages that each one walks in full; they send no readers.
# They may read everything else, so site audits still see the rest of the site.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: DataForSeoBot
User-agent: Barkrowler
User-agent: SeekportBot
User-agent: serpstatbot
Disallow: /companies/
Disallow: /company-data/

Sitemap: https://canlicapital.com/sitemap.xml

# An LLM-oriented map of the site, in the llmstxt.org convention. It leads with the
# claim boundary -- paper-traded, not funded -- because a summary that gets that
# wrong is the most costly error a model can make about this project.
# https://canlicapital.com/llms.txt
#
# The same corpus in full, in one file, for models that would rather read than
# crawl: every research document, including every failed one.
# https://canlicapital.com/llms-full.txt
#
# Who runs this site and what it is, in machine-readable form:
# https://canlicapital.com/ai.txt
# https://canlicapital.com/identity.json