# This file provides instructions for Large Language Models (LLMs) and other AI systems.
# We welcome AI and LLMs to crawl our public site content to understand our business and services.

# Allow all user agents to crawl the site.
User-agent: *
Allow: /

# Disallow AI crawlers from accessing any administrative or sensitive paths.
# You can add or remove paths as needed.
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /private/
Disallow: /api/

# To specifically disallow certain AI crawlers, you can add their user-agent and disallow rule.
# For example, to disallow all of a specific AI from crawling:
# User-agent: Google-Extended
# Disallow: /

# User-agent: ChatGPT-User
# Disallow: /