diff --git a/static/robots.txt b/static/robots.txt index 70b781e..d49c9b4 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,3 +1,11 @@ +User-agent: SemrushBot +Disallow: / +User-agent: BLEXBot +Disallow: / +User-agent: AhrefsBot +Disallow: / +User-agent: MJ12bot +Disallow: / User-agent: * Allow: /