{
  "schema_version": "2026-07-09",
  "name": "Spala AI Crawler Access Policy",
  "purpose": "Machine-readable robots.txt and AI crawler access policy for public Spala discovery pages.",
  "boundary": "Crawler access applies only to public pages. It does not expose private source code, private repos, internal IP addresses, customer data, secrets, or private implementation architecture, and it does not guarantee crawling, indexing, ranking, or citation.",
  "robots_txt": "https://spala.ai/robots.txt",
  "sitemap": "https://spala.ai/sitemap.xml",
  "ai_sitemap": "https://spala.ai/sitemap-ai.xml",
  "allowed_user_agents": [
    {
      "provider": "OpenAI",
      "userAgent": "GPTBot",
      "purpose": "OpenAI crawler used for model improvement/training contexts.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "provider": "OpenAI",
      "userAgent": "OAI-SearchBot",
      "purpose": "OpenAI search crawler used for ChatGPT search/discovery surfaces.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "provider": "OpenAI",
      "userAgent": "ChatGPT-User",
      "purpose": "OpenAI user-triggered fetcher for user requests.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://developers.openai.com/api/docs/bots"
    },
    {
      "provider": "Anthropic",
      "userAgent": "ClaudeBot",
      "purpose": "Anthropic crawler for model utility and safety data collection.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "provider": "Anthropic",
      "userAgent": "Claude-User",
      "purpose": "Anthropic user-triggered fetcher for Claude user requests.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "provider": "Anthropic",
      "userAgent": "Claude-SearchBot",
      "purpose": "Anthropic crawler for Claude search result quality and indexing.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "provider": "Perplexity",
      "userAgent": "PerplexityBot",
      "purpose": "Perplexity crawler for surfacing and linking websites in Perplexity search results.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "provider": "Perplexity",
      "userAgent": "Perplexity-User",
      "purpose": "Perplexity user-triggered fetcher for user-requested answers.",
      "publicStatus": "allowed_in_robots_txt",
      "source": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    }
  ],
  "agent_guidance": [
    "Use robots.txt to confirm crawler access.",
    "Use sitemap.xml and sitemap-ai.xml for public page discovery.",
    "Use llms.txt, llms-full.txt, agents.md, .well-known/agent.json, and .well-known/mcp.json for agent context.",
    "Do not infer private source access from crawler access.",
    "Do not claim AI citation, ranking, or indexing has happened unless current external evidence proves it."
  ],
  "ai_discovery_health": "https://spala.ai/ai-discovery-health.json",
  "ai_discovery_health_summary": "2026-07-09 live check: 15/15 core public AI discovery endpoints healthy and 12/12 target AI/search user agents allowed in robots.txt. Discoverability evidence only; not indexing, citation, ranking, or production-trust proof.",
  "schema_health": "https://spala.ai/schema-health.json",
  "schema_health_summary": "2026-07-09 live check: 5/5 standalone JSON-LD schema files reachable and parseable; 5/5 checked public pages had parseable embedded JSON-LD. Structured-data hygiene only; not indexing, rich-result, citation, or production-trust proof."
}