diff --git a/frontend/src/app/robots.txt b/frontend/src/app/robots.txt index 333e7eb6..9d2f72be 100644 --- a/frontend/src/app/robots.txt +++ b/frontend/src/app/robots.txt @@ -1,2 +1,6 @@ +User-agent: * +Crawl-delay: 10 +Disallow: /api + User-agent: Bytespider Disallow: /