# robots.txt for www.american.edu
# Use this to block robots in an emergency
# Disallow: / 

# Block OpenAI
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /

User-agent: *
Crawl-delay: 10

Sitemap: https://www.american.edu/sitemap.cfm

#Disallow: /spa/calendar/
#Disallow: /alumni/events/
#Disallow: /cas/calendar/
#Disallow: /kogod/calendar/
#Disallow: /sis/calendar/
#Disallow: /soc/calendar/
#Disallow: /spa/calendar/
#News - high processing pages 
Disallow: /news/all-news-more.cfm
Disallow: /news/archive/all-news.cfm
Disallow: /customcf/email-a-friend.cfm
Disallow: /profiles/admin/
Disallow: /oit/network/Wired-7.cfm 
Disallow: /customcf/au-experts/details.cfm
Disallow: /directory/?
Disallow: /online/online-program/draft-online-program-page.cfm
Disallow: /online/online-program/curriculum_draft.cfm