# References:
# https://en.wikipedia.org/wiki/Robots.txt
# https://support.google.com/webmasters/answer/1061943?hl=en
# https://blogs.bing.com/webmaster/2012/05/03/to-crawl-or-not-to-crawl-that-is-bingbots-question/

Sitemap: https://www.sitepoint.com/sitemap.xml

User-agent: *
Allow: /ads.txt
# Blog Related Examples
Disallow: /examples/
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/cert/
# Discourse-related
Disallow: /community/auth/cas
Disallow: /community/auth/facebook/callback
Disallow: /community/auth/twitter/callback
Disallow: /community/auth/google/callback
Disallow: /community/auth/yahoo/callback
Disallow: /community/auth/github/callback
Disallow: /community/auth/cas/callback
Disallow: /community/assets/browser-update*.js
Disallow: /community/users/
Disallow: /community/u/
Disallow: /community/badges/
Disallow: /community/search
Disallow: /community/search/
Disallow: /community/tags
Disallow: /community/tags/
Disallow: /community/email/
Disallow: /community/session
Disallow: /community/session/
Disallow: /community/admin
Disallow: /community/admin/
Disallow: /community/user-api-key
Disallow: /community/user-api-key/
Disallow: /community/*?api_key*
Disallow: /community/*?*api_key*
Disallow: /community/groups
Disallow: /community/groups/
Disallow: /community/t/*/*.rss
Disallow: /community/tags/*.rss
Disallow: /community/c/*.rss


# Adds a Crawl-delay of 30 to bingbot. Otherwise, identical to the
# above. Requested by Discourse.
User-agent: bingbot
Crawl-delay: 30
Disallow: /ads.txt
# Blog Related Examples
Disallow: /examples/
# SitePoint Premium-related
Disallow: /premium/admin/
Disallow: /premium/l/
Disallow: /premium/cert/
# Discourse-related
Disallow: /community/auth/cas
Disallow: /community/auth/facebook/callback
Disallow: /community/auth/twitter/callback
Disallow: /community/auth/google/callback
Disallow: /community/auth/yahoo/callback
Disallow: /community/auth/github/callback
Disallow: /community/auth/cas/callback
Disallow: /community/assets/browser-update*.js
Disallow: /community/users/
Disallow: /community/u/
Disallow: /community/badges/
Disallow: /community/search
Disallow: /community/search/
Disallow: /community/tags
Disallow: /community/tags/
Disallow: /community/email/
Disallow: /community/session
Disallow: /community/session/
Disallow: /community/admin
Disallow: /community/admin/
Disallow: /community/user-api-key
Disallow: /community/user-api-key/
Disallow: /community/*?api_key*
Disallow: /community/*?*api_key*
Disallow: /community/groups
Disallow: /community/groups/
Disallow: /community/t/*/*.rss
Disallow: /community/tags/*.rss
Disallow: /community/c/*.rss