[choore] Update robots.txt (#3092)
Recategorises a pair of scrapers according to their use.
This commit is contained in:
parent
1de41f64f2
commit
ad93e57d08
|
@ -47,11 +47,10 @@ User-agent: FriendlyCrawler
|
||||||
User-agent: Google-Extended
|
User-agent: Google-Extended
|
||||||
User-agent: GoogleOther
|
User-agent: GoogleOther
|
||||||
User-agent: GPTBot
|
User-agent: GPTBot
|
||||||
|
User-agent: ImagesiftBot
|
||||||
User-agent: img2dataset
|
User-agent: img2dataset
|
||||||
User-agent: omgili
|
User-agent: omgili
|
||||||
User-agent: omgilibot
|
User-agent: omgilibot
|
||||||
User-agent: peer39_crawler
|
|
||||||
User-agent: peer39_crawler/1.0
|
|
||||||
User-agent: PerplexityBot
|
User-agent: PerplexityBot
|
||||||
User-agent: YouBot
|
User-agent: YouBot
|
||||||
Disallow: /
|
Disallow: /
|
||||||
|
@ -60,9 +59,10 @@ Disallow: /
|
||||||
User-agent: AwarioRssBot
|
User-agent: AwarioRssBot
|
||||||
User-agent: AwarioSmartBot
|
User-agent: AwarioSmartBot
|
||||||
User-agent: DataForSeoBot
|
User-agent: DataForSeoBot
|
||||||
User-agent: ImagesiftBot
|
|
||||||
User-agent: magpie-crawler
|
User-agent: magpie-crawler
|
||||||
User-agent: Meltwater
|
User-agent: Meltwater
|
||||||
|
User-agent: peer39_crawler
|
||||||
|
User-agent: peer39_crawler/1.0
|
||||||
User-agent: PiplBot
|
User-agent: PiplBot
|
||||||
User-agent: scoop.it
|
User-agent: scoop.it
|
||||||
User-agent: Seekr
|
User-agent: Seekr
|
||||||
|
|
Loading…
Reference in New Issue