# Imagera AI robots.txt
# https://imagera.ai
#
# GOVERNING PRINCIPLE — robots.txt controls CRAWLING, never INDEXING.
# If a URL must not be indexed, it gets `X-Robots-Tag: noindex` from worker.ts
# and stays CRAWLABLE so the crawler can actually read that header. Adding a
# Disallow on top of a noindex cancels the noindex and strands the URL as a
# permanent URL-only "Indexed, though blocked by robots.txt" entry.
# Disallow is reserved for surfaces that are private (/vault, /admin, /auth/)
# or of unbounded cardinality (tracking params).
#
# Updated: July 2026 - Applied the principle above: dropped the ?tag=/?sort=/
# ?filter= facet blocks and the /{image,video,audio,avatar}/*/studio blocks,
# all of which sat on top of a worker.ts noindex; unblocked AhrefsBot and
# SemrushBot (our own paid tooling, blocked by an undocumented 2025-09 bulk
# edit); closed RFC 9309 §2.2.1 inheritance gaps in the Googlebot group.
# Updated: April 2026 - Removed disallow on content hub paths so Google can
# read the in-app noindex meta tag (set in ContentIndex via NOINDEX_CONTENT_TYPES)
# and drop empty hubs cleanly. Hubs with real published children (e.g. /glossary
# has 15 entries, /tutorials has 8) MUST be crawlable so Google can reach the
# child pages.
# See also: https://imagera.ai/llms.txt for LLM-specific content
# ============================================
# Allow all search engine crawlers (Default)
# ============================================
User-agent: *
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /auth/callback
Disallow: /api/
Disallow: /subscription
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /api/*.json
# NOTE — bounded in-app facets (?tag=, ?sort=, ?filter=, ?category=) are NOT
# blocked, deliberately. worker.ts (NOINDEX_QUERY_PARAMS ~:275, header set
# ~:762) serves `X-Robots-Tag: noindex, nofollow` on every one of them:
#   curl -I "https://imagera.ai/blog?tag=..." -> x-robots-tag: noindex, nofollow
# A Disallow here would CANCEL that header — a crawler not allowed to fetch the
# URL can never read the noindex it carries, so the URL lingers forever as a
# URL-only "Indexed, though blocked by robots.txt" entry. Crawlable + noindex is
# strictly better: fetched once, understood, dropped permanently.
# Evidence (GSC 2026-04-30..07-30): the blocked ?tag= URLs were stranded at
# positions 52.5 / 62.9; the never-blocked ?category= URLs — same noindex
# header, no Disallow — sat at positions 3.0-8.7 and process normally.
# Crawl cost is small: facets are bounded and NOT server-rendered (zero ?tag=
# anchors exist in the prerendered HTML). Campaign duplicates use the clean
# canonical, per Google Search Central canonicalization guidance.
# Campaign URLs stay crawlable so their canonical tags can consolidate them.
# Sharing parameters remain excluded.
Disallow: /*?*share=
Disallow: /*?*shareId=
Disallow: /*?*ref=
Disallow: /*?*trk=
# Note: content hub paths (/use-cases, /features, /apps, /loras, /models,
# /styles, /templates, /glossary, /showcase) are crawlable. Empty hubs are
# noindexed in-app via ContentIndex.tsx NOINDEX_CONTENT_TYPES so Google can
# crawl them, see the noindex tag, and drop them cleanly. Hubs with published
# children must be crawlable so child pages can be discovered and indexed.
Allow: /cdn-cgi/image/
Disallow: /cdn-cgi/
# Studio pages — SPA shell only (behind auth), no crawlable content.
# These are NOT blocked in robots.txt. The enforcing mechanism is
# worker.ts shouldNoindex() (~:236), which serves
# `X-Robots-Tag: noindex, nofollow` on ANY nested studio surface via
# /\/studio(\/|$)/ and covers all 95 studio routes. That is strictly broader
# than the old `/{image,video,audio,avatar}/*/studio` Disallows, which silently
# missed /llm-arena/studio, /image-arena/studio, /ai-smart-director/studio and
# the nested music-factory studios.
# Why removed rather than copied into the Googlebot group (RFC 9309 §2.2.1 —
# groups do not inherit, so the rules were inert for Googlebot anyway):
# blocking a studio would CANCEL its noindex, exactly like the ?tag= trap above.
# Google cannot read a noindex on a URL it is not allowed to fetch, so ~95
# app-shell URLs would become permanent URL-only index entries. Crawlable +
# noindex lets Google fetch once, see noindex, and drop them cleanly.
# Studios that opted INTO being indexable (SSOT: worker-lib/indexable-studios.ts)
# need no Allow line now that nothing blocks them.
# ============================================
# Google Bots (Priority Crawlers)
# ============================================
User-agent: Googlebot
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /subscription
Disallow: /admin
Disallow: /dashboard
# RFC 9309 §2.2.1 — this group applies to Googlebot EXCLUSIVELY; it inherits
# nothing from `*`. These two were in `*` and `bingbot` but missing here, so
# Googlebot alone could crawl the JSON API surface. /api/ 301s and
# /api/*.json 404s — zero /api/ URLs in the 400-URL sitemap and zero in GSC.
Disallow: /api/
Disallow: /api/*.json
# Bounded in-app facets (?tag=, ?sort=, ?filter=, ?category=) are intentionally
# NOT blocked — see the full rationale in the `*` group above. They carry
# `X-Robots-Tag: noindex, nofollow` from worker.ts, which Googlebot can only
# act on if it is allowed to fetch the URL.
# Campaign URLs stay crawlable so their canonical tags can consolidate them.
# Sharing parameters remain excluded.
Disallow: /*?*share=
Disallow: /*?*shareId=
Disallow: /*?*ref=
Disallow: /*?*trk=
# Content hub paths are crawlable — see comment in default block above
Allow: /cdn-cgi/image/
Disallow: /cdn-cgi/
User-agent: Googlebot-Image
Allow: /
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /optimized/
Disallow: /vault
Disallow: /admin
User-agent: Googlebot-Video
Allow: /
# Google Extended (for Bard/Gemini AI)
User-agent: Google-Extended
Allow: /
Allow: /resources
Allow: /prompt-guide
Allow: /support
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
Crawl-delay: 2
# ============================================
# AI Search Engines & Language Model Bots
# ============================================
# OpenAI GPT Bot (Model Training)
User-agent: GPTBot
Allow: /
Allow: /resources
Allow: /prompt-guide
Allow: /support
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /subscription
Disallow: /admin
Crawl-delay: 2
# OpenAI Search Bot (Search Results - IMPORTANT for AI search visibility)
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /subscription
Disallow: /admin
# ChatGPT User Agent (User-initiated browsing)
User-agent: ChatGPT-User
Allow: /
Allow: /resources
Allow: /prompt-guide
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Common Crawl Bot
User-agent: CCBot
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Anthropic Claude Bot (Training)
User-agent: ClaudeBot
Allow: /
Allow: /resources
Allow: /prompt-guide
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
Crawl-delay: 2
User-agent: anthropic-ai
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
User-agent: Claude-Web
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Anthropic Claude Search Bot (Search results — AI search visibility)
User-agent: Claude-SearchBot
Allow: /
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Anthropic Claude User Agent (User-triggered browsing)
User-agent: Claude-User
Allow: /
Allow: /resources
Allow: /prompt-guide
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Perplexity AI Bot (Index Building)
User-agent: PerplexityBot
Allow: /
Allow: /resources
Allow: /prompt-guide
Allow: /support
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
Crawl-delay: 2
# Perplexity User Agent (Human-triggered visits)
User-agent: Perplexity-User
Allow: /
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Meta AI Bot
User-agent: FacebookBot
Allow: /
Disallow: /vault
Disallow: /admin
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
# Cohere AI Bot
User-agent: cohere-ai
Allow: /
Disallow: /vault
Disallow: /admin
# Amazon Bots
User-agent: Amazonbot
Allow: /
Disallow: /vault
Disallow: /admin
# Apple Bots
User-agent: Applebot
Allow: /
Disallow: /vault
Disallow: /admin
User-agent: Applebot-Extended
Allow: /
Disallow: /admin
# ============================================
# Other Major Search Engines
# ============================================
# Bing Bot — CRITICAL: bingbot-specific section overrides wildcard (*) entirely.
# All Disallow rules from wildcard MUST be duplicated here or bingbot ignores them.
User-agent: bingbot
Allow: /
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /auth/callback
Disallow: /api/
Disallow: /subscription
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /api/*.json
# NOTE — bounded in-app facets (?tag=, ?sort=, ?filter=, ?category=) are NOT
# blocked, deliberately. worker.ts (NOINDEX_QUERY_PARAMS ~:275, header set
# ~:762) serves `X-Robots-Tag: noindex, nofollow` on every one of them:
#   curl -I "https://imagera.ai/blog?tag=..." -> x-robots-tag: noindex, nofollow
# A Disallow here would CANCEL that header — a crawler not allowed to fetch the
# URL can never read the noindex it carries, so the URL lingers forever as a
# URL-only "Indexed, though blocked by robots.txt" entry. Crawlable + noindex is
# strictly better: fetched once, understood, dropped permanently.
# Evidence (GSC 2026-04-30..07-30): the blocked ?tag= URLs were stranded at
# positions 52.5 / 62.9; the never-blocked ?category= URLs — same noindex
# header, no Disallow — sat at positions 3.0-8.7 and process normally.
# Crawl cost is small: facets are bounded and NOT server-rendered (zero ?tag=
# anchors exist in the prerendered HTML). Campaign duplicates use the clean
# canonical, per Google Search Central canonicalization guidance.
# Campaign URLs stay crawlable so their canonical tags can consolidate them.
# Sharing parameters remain excluded.
Disallow: /*?*share=
Disallow: /*?*shareId=
Disallow: /*?*ref=
Disallow: /*?*trk=
# Note: content hub paths (/use-cases, /features, /apps, /loras, /models,
# /styles, /templates, /glossary, /showcase) are crawlable. Empty hubs are
# noindexed in-app via ContentIndex.tsx NOINDEX_CONTENT_TYPES so Google can
# crawl them, see the noindex tag, and drop them cleanly. Hubs with published
# children must be crawlable so child pages can be discovered and indexed.
Allow: /cdn-cgi/image/
Disallow: /cdn-cgi/
# Studio pages are not blocked here either — see the full rationale in the `*`
# group above. worker.ts serves `X-Robots-Tag: noindex, nofollow` on every
# studio route, and blocking them would cancel that header.
# Bing Preview
User-agent: BingPreview
Allow: /
Disallow: /admin
# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Disallow: /vault
Disallow: /admin
# Yandex Bot — June 2026: YandexBot was 83.5% of ALL pageviews (353K pages/8d,
# Cloudflare zone analytics) with zero referral revenue. Crawl-delay raised to
# 10 (Yandex honors it); rate is also clamped in Yandex.Webmaster if needed.
User-agent: YandexBot
Allow: /
Disallow: /vault
Disallow: /admin
Crawl-delay: 10
# Baidu Bot
User-agent: Baiduspider
Allow: /
Disallow: /vault
Disallow: /admin
Crawl-delay: 2
# ============================================
# Social Media Bots (For Link Previews)
# ============================================
# Twitter/X Bot
User-agent: Twitterbot
Allow: /
# LinkedIn Bot
User-agent: LinkedInBot
Allow: /
# Pinterest Bot
User-agent: Pinterestbot
Allow: /
Allow: /optimized/
# WhatsApp Bot
User-agent: WhatsApp
Allow: /
# Telegram Bot
User-agent: TelegramBot
Allow: /
# Slack Bot
User-agent: Slackbot
Allow: /
# Discord Bot
User-agent: Discordbot
Allow: /
# ============================================
# Block Malicious/Scraper Bots
# ============================================
# ── Owned SEO tooling — deliberately ALLOWED ──
# History: 3378ef91 (2025-08-05) authored these as "# SEO and monitoring bots"
# with `Allow: /` + `Crawl-delay: 10`. 28eb6b97 (2025-09-18) — a ~200-file bulk
# commit titled only "updated" that wholesale-replaced robots.txt — refiled them
# under "# Block malicious bots" as `Disallow: /`. No rationale was recorded in
# any commit message or comment, and they were grouped with MJ12bot/BLEXBot/
# DotBot: the signature of a copy-pasted boilerplate blocklist, not a decision.
# Restoring the authored state, because the block had a real cost and no
# benefit: we pay for Ahrefs, and Disallow made our own site un-auditable —
# backlink/site-audit data could not be gathered for our own domain.
# It protected nothing. Ahrefs/Semrush read only public pages (the same HTML
# Googlebot gets), and blocking their crawler does NOT hide our backlink
# profile — backlinks are discovered on the LINKING sites, not on ours. So the
# block blinded us to our own data while competitors' view was unaffected.
# Crawl-delay: 10 (both honor it) keeps the bandwidth cost bounded, and the
# private surfaces below are disallowed exactly as for every other crawler.
User-agent: AhrefsBot
Allow: /
Disallow: /vault
Disallow: /auth/
Disallow: /subscription
Disallow: /admin
Disallow: /dashboard
Crawl-delay: 10
User-agent: SemrushBot
Allow: /
Disallow: /vault
Disallow: /auth/
Disallow: /subscription
Disallow: /admin
Disallow: /dashboard
Crawl-delay: 10
# ── Genuine scraper / aggressive-crawler blocks ──
User-agent: DotBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MegaIndex
Disallow: /
# Bad/spam bots
User-agent: rogerbot
Disallow: /
User-agent: proximic
Disallow: /
User-agent: SiteBot
Disallow: /
User-agent: spbot
Disallow: /
User-agent: TurnitinBot
Disallow: /
# ============================================
# Additional AI Search Bots (2025-2026)
# ============================================
# You.com Search Bot
User-agent: YouBot
Allow: /
Disallow: /vault
Disallow: /admin
# Brave Search Bot
User-agent: BraveBot
Allow: /
Disallow: /vault
Disallow: /admin
# Mojeek Bot
User-agent: MojeekBot
Allow: /
Disallow: /vault
Disallow: /admin
# Microsoft Copilot Bot
User-agent: MicrosoftBot
Allow: /
Disallow: /vault
Disallow: /admin
# Google Gemini API Access
User-agent: Google-InspectionTool
Allow: /
Disallow: /admin
# Google Gemini Deep Research (User-triggered deep research)
User-agent: Gemini-Deep-Research
Allow: /
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# Mistral AI Bot
User-agent: MistralBot
Allow: /
Allow: /llms.txt
Disallow: /vault
Disallow: /admin
# xAI Grok Bot (Training/Index)
User-agent: GrokBot
Allow: /
Allow: /llms.txt
Disallow: /vault
Disallow: /admin
# xAI Grok Deep Search (User-triggered deep research)
User-agent: Grok-DeepSearch
Allow: /
Allow: /llms.txt
Disallow: /vault
Disallow: /admin
# Bytedance / TikTok Bot
User-agent: Bytespider
Allow: /
Allow: /llms.txt
Disallow: /vault
Allow: /author/
Disallow: /auth/
Disallow: /admin
# AI21 Labs Bot
User-agent: AI21Bot
Allow: /
Disallow: /vault
Disallow: /admin
# Inflection AI (Pi)
User-agent: InflectionBot
Allow: /
Disallow: /vault
Disallow: /admin
# Neeva Search Bot (acquired by Snowflake)
User-agent: NeevaBot
Allow: /
Disallow: /vault
Disallow: /admin
# Exa AI (semantic search)
User-agent: ExaBot
Allow: /
Allow: /llms.txt
Disallow: /vault
Disallow: /admin
# ============================================
# Sitemap & LLM Content
# ============================================
Sitemap: https://imagera.ai/sitemap.xml
Sitemap: https://imagera.ai/video-sitemap.xml
# LLM-specific content descriptor (llmstxt.org standard)
# Provides structured content for AI models to understand the site
# https://imagera.ai/llms.txt
# Host declaration (preferred domain)
Host: https://imagera.ai