SuperSeriousBusiness
/
GoToSocial
mirror of https://github.com/superseriousbusiness/gotosocial
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

[chore] Update robots.txt, give chatgpt the middle finger (#2085)

This commit is contained in:
tobi 2023-08-08 13:16:34 +02:00 committed by GitHub
parent 9df4d38c43
commit 4b05dcde43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
internal/web/robots.go
View File

@ -27,29 +27,43 @@ const (
robotsPath = "/robots.txt" robotsPath = "/robots.txt"
robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta
robotsTxt = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go robotsTxt = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go
# more info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro # More info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro
# Before we commence, a giant fuck you to ChatGPT in particular.
# https://platform.openai.com/docs/gptbot
User-agent: GPTBot
Disallow: /
# Rules for everything else.
User-agent: * User-agent: *
Crawl-delay: 500 Crawl-delay: 500
# api stuff
# API endpoints.
Disallow: /api/ Disallow: /api/
# auth/login stuff
# Auth/login endpoints.
Disallow: /auth/ Disallow: /auth/
Disallow: /oauth/ Disallow: /oauth/
Disallow: /check_your_email Disallow: /check_your_email
Disallow: /wait_for_approval Disallow: /wait_for_approval
Disallow: /account_disabled Disallow: /account_disabled
# well known stuff
# Well-known endpoints.
Disallow: /.well-known/ Disallow: /.well-known/
# files
# Fileserver/media.
Disallow: /fileserver/ Disallow: /fileserver/
# s2s AP stuff
# Fedi S2S API endpoints.
Disallow: /users/ Disallow: /users/
Disallow: /emoji/ Disallow: /emoji/
# panels
# Settings panels.
Disallow: /admin Disallow: /admin
Disallow: /user Disallow: /user
Disallow: /settings/ Disallow: /settings/
# domain blocklist
# Domain blocklist.
Disallow: /about/suspended` Disallow: /about/suspended`
) )