diff options
Diffstat (limited to 'public/robots.txt')
| -rw-r--r-- | public/robots.txt | 23 |
1 files changed, 3 insertions, 20 deletions
diff --git a/public/robots.txt b/public/robots.txt index 582552d..1200905 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,21 +1,4 @@ +# Learn more about robots.txt: https://www.robotstxt.org/robotstxt.html User-agent: * -Disallow: /admin -Disallow: /login -Disallow: /js -Disallow: /css - -User-Agent: GPTBot -Disallow: / - -User-Agent: ChatGPT-User -Disallow: / - -User-Agent: Google-Extended -Disallow: / - -User-Agent: CCBot -Disallow: / - -User-Agent: PerplexityBot -Disallow: / - +# 'Disallow' with an empty value allows all paths to be crawled +Disallow: |