diff --git a/sapl/templates/robots.txt b/sapl/templates/robots.txt new file mode 100644 index 000000000..dad99b301 --- /dev/null +++ b/sapl/templates/robots.txt @@ -0,0 +1,36 @@ +User-agent: semrushbot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +User-agent: SemrushBot +Disallow: / + +User-agent: AhrefsBot +Disallow: / + +User-agent: spbot +Disallow: / + +User-agent: dotbot +Disallow: / + +User-agent: ahrefsbot +Disallow: / + +User-agent: mojeekbot +Disallow: / + +User-agent: yandex +Disallow: / + +User-agent: baiduspider +Disallow: / + +User-agent: * +# Crawl Delay 60 seg. +Crawl-delay: 60 +Disallow: /materia/docacessorio/pdf/* +Disallow: /materia/docacessorio/zip/* +Disallow: /private/ \ No newline at end of file diff --git a/sapl/test_urls.py b/sapl/test_urls.py index 78d0388f4..8a5878e72 100644 --- a/sapl/test_urls.py +++ b/sapl/test_urls.py @@ -20,7 +20,6 @@ _lista_urls = lista_urls() def create_perms_post_migrate(sapl_app_config): - searched_perms = list() # The codenames and ctypes that should exist. ctypes = set() @@ -74,7 +73,6 @@ btn_login = ('