diff --git a/tildes/static/robots.txt b/tildes/static/robots.txt index a3b8c00..47bb3cd 100644 --- a/tildes/static/robots.txt +++ b/tildes/static/robots.txt @@ -1,6 +1,12 @@ +User-agent: AhrefsBot +Disallow: / + User-agent: BLEXBot Disallow: / +User-agent: Buck +Disallow: / + User-agent: dotbot Disallow: / @@ -9,3 +15,6 @@ Disallow: / User-agent: MegaIndex Disallow: / + +User-agent: SemrushBot +Disallow: /